Bump release version to 0.9.72rc105

This reverts commit da1665acb3.

.github/CONTRIBUTING.md

@@ -12,7 +12,7 @@
 - https://github.com/ArchiveBox/ArchiveBox/issues
 - https://github.com/ArchiveBox/ArchiveBox/pulls
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup
 
 ### Development Setup
 

.github/STAGING.md

@@ -0,0 +1,42 @@
+# Stable releases require live dev acceptance
+
+The order is **dev CI → dev image publication → automatic Cabbage deployment →
+real capture and browser acceptance → stable publication**. Never
+substitute a published PyPI wheel, healthy container, or local build for this
+acceptance. The release workflow blocks *before* a stable upload unless Cabbage
+has successful evidence from `staging-acceptance.yml` matching all tracked
+source and dependency pins. Only ArchiveBox's own version fields may change
+between dev and stable.
+
+The staging workflow observes the existing image watcher; it cannot deploy or
+restart the application. On Cabbage, install `bin/staging-acceptance-host.sh`
+as root-owned mode 0755 `/usr/local/sbin/archivebox-staging-acceptance`. Its hash
+is checked against the candidate. A dedicated SSH key must use
+`restrict,command="/usr/local/sbin/archivebox-staging-acceptance"` in authorized
+keys. It accepts only `inspect <source-sha>` and `capture <source-sha>`; it cannot
+run caller-provided commands or archive arbitrary URLs.
+
+The `staging` GitHub environment is restricted to dev/main branches. It contains
+`STAGING_SSH_KEY` and `STAGING_KNOWN_HOSTS`, with independently verified host keys.
+Only trusted release events use the private ugNAS runner route. Never expose
+these credentials to PR jobs or copy an operator's existing root key.
+
+Acceptance creates one tagged background crawl per source through the normal
+add/runner path, enabling every snapshot extractor, including TLSNotary. It
+checks installed pins, completed outputs and their files, unexpected/missing
+plugin results, host OOM kills, and real rendered snapshot pages. For this
+release, the explicitly accepted TLSNotary failures and exact missing-Claude-auth
+diagnostics are retained in the report's `limitations` list; they do not disable
+those plugins or excuse missing/truncated successful outputs. Remove the
+TLSNotary exception when its follow-up fix ships. Logs, result inventories and
+screenshots are retained as Actions artifacts. Other failures remain
+failures; do not publish a success deployment record to bypass them. No code
+mounts are allowed. The Cabbage job must pass before its record authorizes a
+stable upload. Queued capture progress is polled; failed tests are not retried.
+
+Browser acceptance follows the default WACZ viewer into the captured document,
+matching its replay base and original URL and checking that its body is visible
+and contains substantial text. An attached viewer iframe can still be
+`about:blank`, so its presence alone cannot prove replay works. Extracted display
+titles and the captured document's `<title>` may legitimately differ; neither
+title equality nor output thumbnails substitute for checking the actual page.

.github/actionlint.yaml

@@ -0,0 +1,3 @@
+self-hosted-runner:
+  labels:
+    - ugnas-linux

.github/dependabot.yml

@@ -15,7 +15,7 @@ updates:
         patterns:
           - "*"
   - package-ecosystem: "npm"
-    directory: "/"
+    directory: "/etc"
     target-branch: "dev"
     schedule:
       interval: "monthly"

.github/lychee.toml

@@ -0,0 +1,38 @@
+# Documentation destinations only: do not crawl archived URLs, app routes, or image assets.
+# Manual run from the repository root with Lychee 0.24.2:
+# git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > /tmp/archivebox-link-inputs.txt
+# lychee --config .github/lychee.toml --root-dir "$PWD" --files-from /tmp/archivebox-link-inputs.txt
+# Explicit tracked inputs include README symlinks and let errors in one file leave others checkable.
+include_fragments = "anchor-only"
+include_verbatim = true
+max_retries = 0
+max_concurrency = 4
+timeout = 20
+no_progress = true
+hidden = true
+extensions = ["md", "html", "py", "rst", "yml", "yaml", "sh", "toml"]
+exclude_path = ['(^|/)(tests|data|apidocs|node_modules|\.git)(/|$)']
+# Includes override this exclude; this is a documentation-link allowlist.
+exclude = ['.*']
+include = [
+ '^https://github\.com/ArchiveBox/ArchiveBox/blob/[^/]+/.*\.md(?:#|$)',
+ '^https://raw\.githubusercontent\.com/wiki/ArchiveBox/ArchiveBox/',
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox(?:/wiki(?:/|#|$)|/?#)',
+ '^https?://docs\.archivebox\.io(?:/|$)',
+ '^file:.*\.md(?:#|$)',
+ '^file:.*/docs/[^/.#]+(?:#|$)',
+]
+fallback_extensions = ["md"]
+remap = [
+ # Lychee's Markdown slugger drops variation selectors. Check these against rendered HTML.
+
+ '^file:.*/README\.md(#%EF%B8%8F-cli-usage)$ https://github.com/ArchiveBox/ArchiveBox${1}',
+ '^file:.*/docs/Install\.md(#option-a-docker--docker-compose-setup-%EF%B8%8F)$ https://github.com/ArchiveBox/ArchiveBox/wiki/Install${1}',
+ # The wiki accepts this case alias, but raw file paths are case-sensitive.
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/Setting-up-Storage$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/Setting-Up-Storage.md',
+
+ # Bare missing wiki pages silently redirect to Home; raw pages return an actual 404.
+ # Keep fragment URLs on the rendered wiki to catch published heading drift.
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/([^#/?]+)$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/${1}.md',
+]
+preprocess = { command = "bin/lychee_preprocess.py" }

.github/pages/README.md

@@ -65,10 +65,15 @@ Build: `uv run --no-project python .github/pages/site.py render _site --source p
 Check the built site with real Chromium (also run by Pages CI):
 
 ```sh
-uv run --no-config --no-project --with playwright==1.63.0 playwright install chromium
+export ABXPKG_LIB_DIR=/tmp/archivebox-site-browser
+CHROME_SCHEMA="$(uv run python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
+uv run abxpkg env --install --deps-from="$CHROME_SCHEMA:required_binaries"
+export CHROME_BINARY="$ABXPKG_LIB_DIR/env/bin/chromium"
 uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py SITE_OUTPUT --evidence /tmp/site-evidence
 ```
 
+The browser comes from the normal abxpkg provider/cache path; Playwright drives
+that resolved executable instead of installing a second Chromium build.
 Use this workflow's `SITE_OUTPUT` directory. The check covers desktop/mobile
 layout, keyboard dismissal, no-JavaScript links, local resources and visible
 images, and uploads its screenshots as `site-verification` for 14 days.

.github/pages/artifacts.py

@@ -14,7 +14,7 @@ def trusted(run, repo, workflow, branch, successful=True):
     return (
         (run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
         and run["head_branch"] == branch
-        and run["event"] in ("push", "workflow_dispatch")
+        and run["event"] in ("push", "workflow_dispatch", "schedule")
         and run["path"] == f".github/workflows/{workflow}"
         and (not successful or run["conclusion"] == "success")
     )

.github/pages/base/footer-start.html

@@ -10,4 +10,4 @@ Also review the inline navbar/footer in abx-dl/website/index.html and
 -->
 <footer class="abx-footer" aria-label="ArchiveBox ecosystem">
   <div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
-  <div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps &amp; tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS &amp; iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS &amp; iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated
+  <div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps &amp; tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS &amp; iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS &amp; iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated

.github/pages/verify.py

@@ -2,6 +2,7 @@
 
 import argparse
 import json
+import os
 import re
 from functools import partial
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
@@ -75,7 +76,7 @@ def verify(output, evidence):
     origin = f"http://127.0.0.1:{server.server_port}"
     try:
         with sync_playwright() as playwright:
-            browser = playwright.chromium.launch(ignore_default_args=["--hide-scrollbars"])
+            browser = playwright.chromium.launch(executable_path=os.environ.get("CHROME_BINARY"), ignore_default_args=["--hide-scrollbars"])
             for name in config["pages"]:
                 route = name.removesuffix("index.html")
                 page = browser.new_page(reduced_motion="reduce")

.github/scripts/discover_test_matrix.py

@@ -1,37 +1,79 @@
 #!/usr/bin/env python3
-"""Discover every ArchiveBox test file and expose it as one CI job."""
+"""Discover every test once, batching measured short files."""
 
 import json
+import os
 from pathlib import Path
 
 
 def main() -> None:
-    root = Path.cwd().resolve()
+    root = Path(__file__).resolve().parents[2]
     archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
 
     if not archivebox_tests:
         raise SystemExit("No ArchiveBox tests discovered")
 
-    matrix: list[dict[str, object]] = []
+    # These are observed per-file pytest seconds, including fixture/startup
+    # time. Shared job setup is paid once per batch. Unknown/long files, optional
+    # dependencies and hosted-only requirements keep independent jobs.
+    durations = json.loads((root / ".github/test-durations.json").read_text())["seconds"]
+    hosted = {path for path in archivebox_tests if "# ci-runner: hosted" in path.read_text().splitlines()[:5]}
+    requested = os.environ.get("TEST_FILE", "")
+    batches: list[list[str]] = []
+    batch: list[str] = []
+    batch_seconds = 0
     for path in archivebox_tests:
         test_path = path.relative_to(root).as_posix()
+        duration = durations.get(test_path, 60)
+        if duration >= 60 or path in hosted or path.stem == "test_auth_ldap" or test_path == requested:
+            batches.append([test_path])
+            continue
+        if batch and (len(batch) == 8 or batch_seconds + duration > 180):
+            batches.append(batch)
+            batch = []
+            batch_seconds = 0
+        batch.append(test_path)
+        batch_seconds += duration
+    if batch:
+        batches.append(batch)
+
+    matrix: list[dict[str, object]] = []
+    for paths in batches:
         matrix.append(
             {
-                "name": f"main/{path.stem}",
-                "paths": [test_path],
-                "paths_arg": test_path,
-                "extra": "ldap" if test_path.endswith("/test_auth_ldap.py") else "",
-                "count": 1,
+                "name": f"main/{Path(paths[0]).stem}" if len(paths) == 1 else f"batch/{Path(paths[0]).stem}",
+                "paths": paths,
+                "paths_arg": " ".join(paths),
+                "extra": "ldap" if paths == ["archivebox/tests/test_auth_ldap.py"] else "",
+                "count": len(paths),
+                "ugnas": False,
             },
         )
 
+    # Ordinary Linux tests can use either pool. Only genuine hosted-only
+    # requirements need a file header; filenames never enter runner policy.
+    # Keep a bounded NAS share so hosted capacity continues working in parallel.
+    # This is queued work, not a worker count: several waves keep the capped
+    # runners occupied while hosted jobs run in parallel.
+    nas_job_limit = int(os.environ.get("UGNAS_CI_MAX_JOBS", "3"))
+    # Keep long jobs early in the hosted queue. Giving the longest jobs to the
+    # CPU-capped NAS instead created a tail after hosted work had finished; use
+    # its extra slots for a bounded share of shorter work without filename rules.
+    matrix.sort(key=lambda item: sum(durations.get(path, 60) for path in item["paths"]), reverse=True)
+    eligible = [item for item in reversed(matrix) if not any(root / path in hosted for path in item["paths"])]
+    for item in eligible[:nas_job_limit]:
+        item["ugnas"] = True
+
     discovered_paths = [path for entry in matrix for path in entry["paths"]]
     if len(discovered_paths) != len(set(discovered_paths)):
         raise SystemExit("Tests were not discovered exactly once")
     if sorted(discovered_paths) != [path.relative_to(root).as_posix() for path in archivebox_tests]:
         raise SystemExit("Discovered test shard coverage does not match test files")
 
-    print(f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs")
+    print(
+        f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs; "
+        f"{sum(item['ugnas'] for item in matrix)} selected for ugNAS, subject to the availability check",
+    )
     print(json.dumps(matrix, separators=(",", ":")))
 
 

.github/test-durations.json

@@ -0,0 +1,176 @@
+{
+  "source_runs": [
+    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37190995942",
+    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37237793977"
+  ],
+  "measurement": "Unbatched files use successful per-file pytest step wall seconds from run 37237793977, including process and fixture startup, on their actual hosted or ugNAS runner. Already-batched files retain prior JUnit measurements plus 9 seconds startup. Queue and shared job setup are excluded; unknown files remain independent.",
+  "seconds": {
+    "archivebox/tests/test_api_discovery_performance.py": 28,
+    "archivebox/tests/test_api_v1_auth_browser_session.py": 17,
+    "archivebox/tests/test_api_v1_auth_check_api_token.py": 15,
+    "archivebox/tests/test_api_v1_auth_get_api_token.py": 16,
+    "archivebox/tests/test_api_v1_cli_add.py": 52,
+    "archivebox/tests/test_api_v1_cli_remove.py": 33,
+    "archivebox/tests/test_api_v1_cli_schedule.py": 29,
+    "archivebox/tests/test_api_v1_cli_search.py": 15,
+    "archivebox/tests/test_api_v1_cli_update.py": 60,
+    "archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 59,
+    "archivebox/tests/test_api_v1_core_any_id.py": 13,
+    "archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 24,
+    "archivebox/tests/test_api_v1_core_archiveresults.py": 19,
+    "archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 106,
+    "archivebox/tests/test_api_v1_core_snapshots.py": 17,
+    "archivebox/tests/test_api_v1_core_snapshots_rss.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id.py": 15,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 16,
+    "archivebox/tests/test_api_v1_core_tags.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 16,
+    "archivebox/tests/test_api_v1_core_tags_autocomplete.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_create.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_search.py": 17,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 69,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 15,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 15,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 16,
+    "archivebox/tests/test_api_v1_crawls_crawls.py": 16,
+    "archivebox/tests/test_api_v1_machine_binaries.py": 16,
+    "archivebox/tests/test_api_v1_machine_binary_binary_id.py": 16,
+    "archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 16,
+    "archivebox/tests/test_api_v1_machine_machine_current.py": 15,
+    "archivebox/tests/test_api_v1_machine_machine_machine_id.py": 16,
+    "archivebox/tests/test_api_v1_machine_machines.py": 15,
+    "archivebox/tests/test_api_v1_personas_personas.py": 15,
+    "archivebox/tests/test_api_v1_personas_sync.py": 16,
+    "archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 27,
+    "archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 19,
+    "archivebox/tests/test_archive_result_service.py": 49,
+    "archivebox/tests/test_auth_ldap.py": 37,
+    "archivebox/tests/test_binary_service.py": 46,
+    "archivebox/tests/test_cli_add_1.py": 75,
+    "archivebox/tests/test_cli_add_2.py": 98,
+    "archivebox/tests/test_cli_add_3.py": 93,
+    "archivebox/tests/test_cli_add_4.py": 94,
+    "archivebox/tests/test_cli_archiveresult.py": 48,
+    "archivebox/tests/test_cli_background_screenshot.py": 49,
+    "archivebox/tests/test_cli_binary.py": 12,
+    "archivebox/tests/test_cli_config.py": 88,
+    "archivebox/tests/test_cli_crawl.py": 81,
+    "archivebox/tests/test_cli_extract.py": 97,
+    "archivebox/tests/test_cli_extract_input.py": 84,
+    "archivebox/tests/test_cli_help.py": 16,
+    "archivebox/tests/test_cli_init.py": 111,
+    "archivebox/tests/test_cli_install.py": 38,
+    "archivebox/tests/test_cli_list.py": 116,
+    "archivebox/tests/test_cli_machine.py": 19,
+    "archivebox/tests/test_cli_manage.py": 32,
+    "archivebox/tests/test_cli_mcp.py": 30,
+    "archivebox/tests/test_cli_oneshot.py": 15,
+    "archivebox/tests/test_cli_persona.py": 56,
+    "archivebox/tests/test_cli_piping.py": 118,
+    "archivebox/tests/test_cli_pluginmap.py": 12,
+    "archivebox/tests/test_cli_process.py": 12,
+    "archivebox/tests/test_cli_remove.py": 46,
+    "archivebox/tests/test_cli_run_1.py": 93,
+    "archivebox/tests/test_cli_run_2.py": 111,
+    "archivebox/tests/test_cli_run_3.py": 51,
+    "archivebox/tests/test_cli_run_4.py": 78,
+    "archivebox/tests/test_cli_schedule.py": 59,
+    "archivebox/tests/test_cli_search.py": 25,
+    "archivebox/tests/test_cli_server.py": 126,
+    "archivebox/tests/test_cli_shell.py": 17,
+    "archivebox/tests/test_cli_snapshot.py": 55,
+    "archivebox/tests/test_cli_status.py": 46,
+    "archivebox/tests/test_cli_tag.py": 12,
+    "archivebox/tests/test_cli_update.py": 62,
+    "archivebox/tests/test_cli_update_reindex_snapshots.py": 84,
+    "archivebox/tests/test_cli_version.py": 81,
+    "archivebox/tests/test_config_DELETE_AFTER.py": 145,
+    "archivebox/tests/test_config_MAX_limits.py": 25,
+    "archivebox/tests/test_config_ONLY_NEW.py": 15,
+    "archivebox/tests/test_config_SAVE_TITLE.py": 59,
+    "archivebox/tests/test_config_URL_filters.py": 15,
+    "archivebox/tests/test_core_config.py": 11,
+    "archivebox/tests/test_core_forms.py": 14,
+    "archivebox/tests/test_crawl_history.py": 60,
+    "archivebox/tests/test_crawl_pause.py": 11,
+    "archivebox/tests/test_crawl_runner.py": 22,
+    "archivebox/tests/test_crawl_runner_2.py": 27,
+    "archivebox/tests/test_crawl_service.py": 19,
+    "archivebox/tests/test_frozen_crawl_config.py": 26,
+    "archivebox/tests/test_hook_result_lifecycle.py": 48,
+    "archivebox/tests/test_hooks.py": 54,
+    "archivebox/tests/test_issue_1896.py": 115,
+    "archivebox/tests/test_machine_models.py": 26,
+    "archivebox/tests/test_machine_service.py": 77,
+    "archivebox/tests/test_migrations_04_to_09.py": 25,
+    "archivebox/tests/test_migrations_07_to_09.py": 86,
+    "archivebox/tests/test_migrations_08_to_09.py": 85,
+    "archivebox/tests/test_migrations_fresh.py": 55,
+    "archivebox/tests/test_misc_checks.py": 12,
+    "archivebox/tests/test_misc_monkey_patches.py": 12,
+    "archivebox/tests/test_opencode_agent_1.py": 73,
+    "archivebox/tests/test_opencode_agent_2.py": 141,
+    "archivebox/tests/test_opencode_agent_3.py": 61,
+    "archivebox/tests/test_opencode_browser_1.py": 139,
+    "archivebox/tests/test_opencode_browser_2.py": 164,
+    "archivebox/tests/test_opencode_browser_3.py": 125,
+    "archivebox/tests/test_orm_helpers.py": 14,
+    "archivebox/tests/test_permissions.py": 12,
+    "archivebox/tests/test_persona_casing.py": 45,
+    "archivebox/tests/test_persona_runtime.py": 30,
+    "archivebox/tests/test_postgres_backend.py": 79,
+    "archivebox/tests/test_process_runtime_paths.py": 20,
+    "archivebox/tests/test_process_service.py": 40,
+    "archivebox/tests/test_proof_previews.py": 24,
+    "archivebox/tests/test_recursive_crawl_1.py": 361,
+    "archivebox/tests/test_recursive_crawl_2.py": 55,
+    "archivebox/tests/test_recursive_crawl_3.py": 67,
+    "archivebox/tests/test_release_notes.py": 12,
+    "archivebox/tests/test_release_source_fingerprint.py": 13,
+    "archivebox/tests/test_release_workflow.py": 15,
+    "archivebox/tests/test_resource_admission.py": 13,
+    "archivebox/tests/test_runner_deletion.py": 91,
+    "archivebox/tests/test_runner_watch.py": 75,
+    "archivebox/tests/test_search.py": 142,
+    "archivebox/tests/test_serve_static.py": 19,
+    "archivebox/tests/test_server_automatic_url.py": 22,
+    "archivebox/tests/test_server_security_browser.py": 121,
+    "archivebox/tests/test_settings_signal_webhooks.py": 13,
+    "archivebox/tests/test_setup_script.py": 12,
+    "archivebox/tests/test_shutdown_util.py": 13,
+    "archivebox/tests/test_snapshot_filesystem_migration.py": 16,
+    "archivebox/tests/test_snapshot_output_stacks.py": 63,
+    "archivebox/tests/test_snapshot_ref_lookup.py": 15,
+    "archivebox/tests/test_snapshot_service.py": 48,
+    "archivebox/tests/test_snapshot_url_length.py": 16,
+    "archivebox/tests/test_tag_service.py": 15,
+    "archivebox/tests/test_takeover_util_1.py": 60,
+    "archivebox/tests/test_takeover_util_2.py": 62,
+    "archivebox/tests/test_takeover_util_3.py": 76,
+    "archivebox/tests/test_takeover_util_4.py": 84,
+    "archivebox/tests/test_test_harness.py": 12,
+    "archivebox/tests/test_ui_add_view.py": 26,
+    "archivebox/tests/test_ui_add_view_runtime.py": 120,
+    "archivebox/tests/test_ui_admin_apitoken.py": 16,
+    "archivebox/tests/test_ui_admin_archiveresult.py": 18,
+    "archivebox/tests/test_ui_admin_config_widget.py": 13,
+    "archivebox/tests/test_ui_admin_crawl.py": 25,
+    "archivebox/tests/test_ui_admin_links.py": 23,
+    "archivebox/tests/test_ui_admin_machine.py": 18,
+    "archivebox/tests/test_ui_admin_persona.py": 19,
+    "archivebox/tests/test_ui_admin_snapshot.py": 138,
+    "archivebox/tests/test_ui_admin_tag.py": 16,
+    "archivebox/tests/test_ui_admin_user.py": 16,
+    "archivebox/tests/test_ui_config_views.py": 20,
+    "archivebox/tests/test_ui_live_progress.py": 121,
+    "archivebox/tests/test_ui_public_snapshot.py": 72,
+    "archivebox/tests/test_ui_savepagenow.py": 31,
+    "archivebox/tests/test_urls.py": 105,
+    "archivebox/tests/test_util.py": 13,
+    "archivebox/tests/test_version_metadata.py": 18,
+    "archivebox/tests/test_zip_preview.py": 29
+  }
+}

.github/workflows/ci.yml

@@ -18,51 +18,64 @@ concurrency:
   cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
 
 jobs:
+  prepare:
+    permissions:
+      contents: write
+    uses: ./.github/workflows/prepare-release.yml
+    secrets: inherit
+
   lint:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Linters
     uses: ./.github/workflows/lint.yml
     secrets: inherit
 
   install-cli-platform:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Install and CLI platform compatibility
     uses: ./.github/workflows/test.yml
     secrets: inherit
 
   tests:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Discovered test matrix
     uses: ./.github/workflows/test-parallel.yml
     secrets: inherit
 
-  documentation:
-    name: Documentation and root tests
-    uses: ./.github/workflows/docs.yml
-    secrets: inherit
-
   codeql:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: CodeQL
     uses: ./.github/workflows/codeql.yml
     secrets: inherit
 
   python-artifacts:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Tested Python artifacts
     uses: ./.github/workflows/pip.yml
     secrets: inherit
 
   docker-digests:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Tested Docker images
     uses: ./.github/workflows/docker.yml
     with:
-      push_digests: false
+      push_digests: ${{ github.event_name == 'push' && (github.ref_name == 'dev' || github.ref_name == 'main') }}
     secrets: inherit
 
   required:
     name: All required CI lanes
-    if: ${{ always() }}
+    if: ${{ always() && needs.prepare.outputs.run_jobs == 'true' }}
     needs:
+      - prepare
       - lint
       - install-cli-platform
       - tests
-      - documentation
       - codeql
       - python-artifacts
       - docker-digests
@@ -73,7 +86,6 @@ jobs:
           LINT_RESULT: ${{ needs.lint.result }}
           INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
           TESTS_RESULT: ${{ needs.tests.result }}
-          DOCUMENTATION_RESULT: ${{ needs.documentation.result }}
           CODEQL_RESULT: ${{ needs.codeql.result }}
           PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
           DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
@@ -82,7 +94,6 @@ jobs:
             "$LINT_RESULT" \
             "$INSTALL_CLI_PLATFORM_RESULT" \
             "$TESTS_RESULT" \
-            "$DOCUMENTATION_RESULT" \
             "$CODEQL_RESULT" \
             "$PYTHON_ARTIFACTS_RESULT" \
             "$DOCKER_DIGESTS_RESULT"

.github/workflows/deploy-publicsite.yml

@@ -52,8 +52,17 @@ jobs:
       # END REPOSITORY-SPECIFIC BUILD
 
       - name: Verify website in Chromium
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/site-browser
         run: |
-          uv run --no-config --no-project --with playwright==1.63.0 playwright install --with-deps chromium
+          # Resolve the same supported browser as captures; Playwright is the
+          # driver only and must not provision a separate browser/runtime.
+          ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
+          CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
+          uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
+          CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
+          export CHROME_BINARY
+          test -x "$CHROME_BINARY"
           uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py "$SITE_OUTPUT" --evidence "$RUNNER_TEMP/site-evidence"
       - name: Save website verification evidence
         if: always()

.github/workflows/docker.yml

@@ -13,6 +13,22 @@ on:
         required: false
         default: true
         type: boolean
+  workflow_dispatch:
+    inputs:
+      source_ref:
+        description: Exact 40-character commit SHA to build
+        required: true
+        type: string
+      push_digests:
+        description: Push validated per-platform images by digest for the release workflow
+        required: false
+        default: true
+        type: boolean
+      full_tests:
+        description: Run storage, CLI, and size validation in addition to release smoke checks
+        required: false
+        default: true
+        type: boolean
 
 env:
   DOCKERHUB_IMAGE: archivebox/archivebox
@@ -42,6 +58,18 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          ref: ${{ inputs.source_ref || github.sha }}
+
+      - name: Verify exact source commit
+        if: github.event_name == 'workflow_dispatch'
+        shell: bash
+        env:
+          SOURCE_REF: ${{ inputs.source_ref || github.sha }}
+        run: |
+          set -Eeuo pipefail
+          [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$(git rev-parse HEAD)" == "$SOURCE_REF" ]]
 
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
@@ -183,7 +211,7 @@ jobs:
           {
             echo 'labels<<EOF'
             echo "org.opencontainers.image.version=${VERSION}"
-            echo "org.opencontainers.image.revision=${GITHUB_SHA}"
+            echo "org.opencontainers.image.revision=${{ inputs.source_ref || github.sha }}"
             echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
             echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
             echo 'EOF'
@@ -203,7 +231,7 @@ jobs:
           labels: ${{ steps.docker_meta.outputs.labels }}
           build-args: |
             ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
+            ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
           pull: true
           platforms: ${{ matrix.platform }}
           # Keep the released image as the one cache authority. A second GHA
@@ -232,7 +260,7 @@ jobs:
           labels: ${{ steps.docker_meta.outputs.labels }}
           build-args: |
             ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
+            ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
           pull: true
           platforms: ${{ matrix.platform }}
           # Test the same layer lineage users receive. Reusing :dev avoids a
@@ -246,33 +274,31 @@ jobs:
         if: inputs.push_digests
         run: echo ${{ steps.docker_build.outputs.digest }}
 
-      - name: Verify OpenCode install with a read-only browser profile
+      - name: Verify optional agent install with a read-only browser profile
         shell: bash
         env:
           IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
         run: |
           set -Eeuo pipefail
           PROFILE_DIR="$(mktemp -d)"
+          # shellcheck disable=SC2016
           "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' \
             --volume "$PROFILE_DIR:/home/archivebox/.config/chromium:ro" \
             "$IMAGE" bash -c '
               set -Eeuo pipefail
               test -w "$XDG_CONFIG_HOME"
               test ! -w "$XDG_CONFIG_HOME/chromium"
-              # Check the shipped executable before the installer can repair
-              # an image that accidentally omitted this opt-in dependency.
-              "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
-              archivebox install opencode
-              "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
+              uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_archivebox_install; verify_archivebox_install()"
             '
 
       - name: Validate exact built image version and commit
-        if: ${{ !inputs.push_digests }}
+        if: inputs.full_tests
         shell: bash
         run: |
           set -Eeuo pipefail
-          IMAGE="${{ matrix.local_tag }}"
-          SHORT_SHA="${GITHUB_SHA::7}"
+          IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}"
+          SHORT_SHA="${{ (inputs.source_ref || github.sha) }}"
+          SHORT_SHA="${SHORT_SHA::7}"
 
           DATA_DIR="$(mktemp -d)"
           "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
@@ -321,6 +347,7 @@ jobs:
           (( COMPRESSED_SIZE < LIMIT ))
 
       - name: Resolve real Docker CLI test dependencies through abxpkg
+        id: mount_dependencies
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash
         run: |
@@ -342,101 +369,143 @@ jobs:
             test -x "$binary"
           done
 
-      - name: Validate Docker UID and GID behavior on real NFS and SMB mounts
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
+      - name: Diagnose failed mount dependency installation
+        if: failure() && steps.mount_dependencies.conclusion == 'failure'
         shell: bash
+        run: |
+          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
+          sudo journalctl --since '-10 minutes' --no-pager \
+            -u smbd -u nmbd -u samba-ad-dc -u nfs-server
+          ps -eo pid,ppid,stat,etimes,wchan:32,comm
+
+      - name: Validate Docker mounts, ports, CLI and compose concurrently
+        if: inputs.full_tests && matrix.platform == 'linux/amd64'
+        shell: bash
+        env:
+          ARCHIVEBOX_IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
         run: |
           set -Eeuo pipefail
-          NFS_SOURCE="/srv/archivebox-ci-nfs"
-          SMB_SOURCE="/srv/archivebox-ci-smb"
-          NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
-          SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
-          mounted_nfs=0
-          mounted_smb=0
+          # These checks own separate collections, Compose projects and ports.
+          # Subshells keep mount cleanup traps and environment changes isolated.
+          validate_mounts() (
+            set -Eeuo pipefail
+            NFS_SOURCE="/srv/archivebox-ci-nfs"
+            SMB_SOURCE="/srv/archivebox-ci-smb"
+            NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
+            SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
+            mounted_nfs=0
+            mounted_smb=0
 
-          cleanup() {
-            local original_status=$? cleanup_status=0
-            trap - EXIT
-            if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
-              cleanup_status=1
+            # Invoked by the subshell EXIT trap.
+            # shellcheck disable=SC2329
+            cleanup() {
+              local original_status=$? cleanup_status=0
+              trap - EXIT
+              if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
+                cleanup_status=1
+              fi
+              if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
+                cleanup_status=1
+              fi
+              if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
+                cleanup_status=1
+              fi
+              if [[ "$original_status" != "0" ]]; then
+                exit "$original_status"
+              fi
+              exit "$cleanup_status"
+            }
+            trap cleanup EXIT
+
+            "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
+            "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
+            "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
+            "$SUDO_BINARY" "$BASH_BINARY" -c \
+              "printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
+            "$SUDO_BINARY" "$BASH_BINARY" -c \
+              "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
+
+            "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
+            "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
+            "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
+            "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
+            "$SYSTEMCTL_BINARY" is-active --quiet smbd.service
+
+            "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
+            mounted_nfs=1
+            "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
+              -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
+            mounted_smb=1
+
+            NFS_TEST_DIR="$NFS_MOUNT" \
+            SMB_TEST_DIR="$SMB_MOUNT" \
+            IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
+              "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
+          )
+          validate_cli() (
+            set -Eeuo pipefail
+            "$MKDIR_BINARY" "${{ github.workspace }}/data"
+            "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
+
+            "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
+            "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
+
+            "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
+            # The argument form above exercises the complete default pipeline.
+            # Keep a real capture for stdin parsing without repeating every plugin.
+            echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add --plugins=wget,hashes
+
+            archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
+            [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
+            [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
+
+            "$DOCKER_BINARY" compose run --rm archivebox init
+            "$DOCKER_BINARY" compose up -d --wait
+            "$DOCKER_BINARY" network connect archivebox_default www-nginx
+            index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
+            admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
+            [[ "$index_html" == *ArchiveBox* ]]
+            [[ "$admin_js" == *window.django* ]]
+
+            "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
+            index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
+            [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
+            "$DOCKER_BINARY" compose down
+          )
+          validate_ports() (
+            IMAGE="$ARCHIVEBOX_IMAGE" "$BASH_BINARY" bin/validate_docker_ports.sh
+          )
+
+          checks=(mounts ports cli)
+          pids=()
+          validate_mounts > "$RUNNER_TEMP/docker-mounts.log" 2>&1 &
+          pids+=("$!")
+          validate_ports > "$RUNNER_TEMP/docker-ports.log" 2>&1 &
+          pids+=("$!")
+          validate_cli > "$RUNNER_TEMP/docker-cli.log" 2>&1 &
+          pids+=("$!")
+          status=0
+          for index in "${!checks[@]}"; do
+            check="${checks[$index]}"
+            echo "::group::Docker $check validation"
+            if wait "${pids[$index]}"; then
+              echo "PASS: Docker $check validation"
+            else
+              echo "::error::Docker $check validation failed"
+              status=1
             fi
-            if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
-              cleanup_status=1
-            fi
-            if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
-              cleanup_status=1
-            fi
-            if [[ "$original_status" != "0" ]]; then
-              exit "$original_status"
-            fi
-            exit "$cleanup_status"
-          }
-          trap cleanup EXIT
-
-          "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-          "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
-          "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-          "$SUDO_BINARY" "$BASH_BINARY" -c \
-            "printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
-          "$SUDO_BINARY" "$BASH_BINARY" -c \
-            "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
-
-          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
-          "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
-          "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
-          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
-          "$SYSTEMCTL_BINARY" is-active --quiet smbd.service
-
-          "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
-          mounted_nfs=1
-          "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
-            -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
-          mounted_smb=1
-
-          NFS_TEST_DIR="$NFS_MOUNT" \
-          SMB_TEST_DIR="$SMB_MOUNT" \
-          IMAGE="${{ matrix.local_tag }}" \
-            "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
+            cat "$RUNNER_TEMP/docker-$check.log"
+            echo "::endgroup::"
+          done
+          exit "$status"
 
       - name: Validate Docker port compatibility
+        if: ${{ !(inputs.full_tests && matrix.platform == 'linux/amd64') }}
         shell: bash
         env:
           IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
         run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
 
-      - name: Validate Docker CLI add/list and compose behavior
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
-        shell: bash
-        env:
-          ARCHIVEBOX_IMAGE: ${{ matrix.local_tag }}
-        run: |
-          set -Eeuo pipefail
-          "$MKDIR_BINARY" "${{ github.workspace }}/data"
-          "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
-
-          "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
-          "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
-
-          "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
-          echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add
-
-          archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
-          [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
-          [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
-
-          "$DOCKER_BINARY" compose run --rm archivebox init
-          "$DOCKER_BINARY" compose up -d --wait
-          "$DOCKER_BINARY" network connect archivebox_default www-nginx
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
-          admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
-          [[ "$index_html" == *ArchiveBox* ]]
-          [[ "$admin_js" == *window.django* ]]
-
-          "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
-          [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
-          "$DOCKER_BINARY" compose down
-
       - name: Validate compressed image size
         if: inputs.full_tests && !inputs.push_digests
         env:
@@ -523,12 +592,17 @@ jobs:
 
       - name: Export digest
         if: inputs.push_digests
+        env:
+          ABX_DL_IMAGE_REF: ${{ steps.abx_dl_image.outputs.image }}
+          ARTIFACT_NAME: ${{ matrix.artifact_name }}
         shell: bash
         run: |
           set -Eeuo pipefail
           mkdir -p /tmp/digests
           digest="${{ steps.docker_build.outputs.digest }}"
           touch "/tmp/digests/${digest#sha256:}"
+          printf '%s\n' '${{ inputs.source_ref || github.sha }}' > /tmp/digests/source-ref
+          printf '%s\n' "$ABX_DL_IMAGE_REF" > "/tmp/digests/abx-dl-image-$ARTIFACT_NAME"
 
       - name: Upload digest
         if: inputs.push_digests

.github/workflows/docs-links.yml

@@ -0,0 +1,38 @@
+name: Documentation link checks
+
+on:
+  schedule:
+    - cron: '17 10 * * *' # Daily at 10:17 UTC, away from the busy top of the hour.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  PYTHONIOENCODING: utf-8
+
+jobs:
+  links:
+    name: Documentation links and anchors
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+      - name: List tracked documentation and source inputs
+        run: |
+          git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > "$RUNNER_TEMP/link-inputs.txt"
+      - uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
+        with:
+          lycheeVersion: v0.24.2
+          args: >-
+            --config .github/lychee.toml
+            --root-dir "${{ github.workspace }}"
+            --files-from "${{ runner.temp }}/link-inputs.txt"
+          fail: true
+          failIfEmpty: true

.github/workflows/docs.yml

@@ -1,7 +1,17 @@
 name: Documentation tests
 
 on:
-  workflow_call:
+  schedule:
+    - cron: '7 5,17 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
 
 env:
   PYTHONIOENCODING: utf-8
@@ -9,7 +19,18 @@ env:
   SHOW_PROGRESS: "False"
 
 jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
   discover:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
     name: Discover documentation examples
     runs-on: ubuntu-24.04
     timeout-minutes: 5
@@ -67,7 +88,7 @@ jobs:
     timeout-minutes: 15
     strategy:
       fail-fast: false
-      max-parallel: 4
+      max-parallel: 8
       matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -146,7 +167,9 @@ jobs:
           fi
       - name: Build the exact checkout's documentation image
         if: ${{ matrix.environment == 'docker' }}
-        run: '"$DOCKER_BINARY" build --tag archivebox/archivebox:dev .'
+        run: |
+          ABX_DL_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(dep.removeprefix("abx-dl==") for dep in tomllib.load(open("pyproject.toml", "rb"))["project"]["dependencies"] if dep.startswith("abx-dl==")))')"
+          "$DOCKER_BINARY" build --build-arg "ABX_DL_IMAGE=archivebox/abx-dl:${ABX_DL_VERSION}" --tag archivebox/archivebox:dev .
       - name: Run the Ubuntu, Docker, or macOS documentation lane
         if: ${{ !contains(fromJSON('["root", "freebsd", "openbsd"]'), matrix.environment) }}
         run: uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
@@ -156,3 +179,17 @@ jobs:
           "$SUDO_BINARY" \
             --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
             "$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
+      - name: Diagnose failed Linux package installation
+        if: failure() && runner.os == 'Linux'
+        run: |
+          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
+          apt-config dump Acquire::Retries Acquire::http::Timeout Acquire::https::Timeout
+          ps -eo pid,ppid,stat,etimes,wchan:32,comm
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, discover, examples]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled work completed successfully.'

.github/workflows/macos.yml

@@ -0,0 +1,48 @@
+name: macOS compatibility
+
+on:
+  schedule:
+    - cron: '51 2,10,18 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
+  install-cli:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
+    uses: ./.github/workflows/test.yml
+    with:
+      platform: macos
+    secrets: inherit
+
+  python-artifacts:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
+    uses: ./.github/workflows/pip.yml
+    with:
+      platforms: '["macos-15"]'
+    secrets: inherit
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, install-cli, python-artifacts]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled macOS coverage completed successfully.'

.github/workflows/pip.yml

@@ -3,6 +3,11 @@ name: Build Pip package
 on:
   workflow_call:
     inputs:
+      platforms:
+        description: Runner matrix for package compatibility
+        required: false
+        default: '["ubuntu-24.04"]'
+        type: string
       full_tests:
         description: Run the full package compatibility matrix instead of one release smoke check
         required: false
@@ -95,7 +100,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-24.04, macos-15]
+        os: ${{ fromJSON(inputs.platforms) }}
         python: ["3.13", "3.14"]
         artifact: [wheel, sdist]
 

.github/workflows/prepare-release.yml

@@ -0,0 +1,55 @@
+name: Prepare release version
+
+on:
+  workflow_call:
+    outputs:
+      run_jobs:
+        description: Whether this revision owns its release version and should run expensive jobs
+        value: ${{ jobs.prepare.outputs.run_jobs }}
+
+permissions:
+  contents: write
+
+jobs:
+  prepare:
+    name: Prepare the version before starting expensive jobs
+    permissions:
+      contents: write
+    runs-on: ubuntu-24.04
+    outputs:
+      run_jobs: ${{ github.event_name != 'push' || steps.channel.outputs.stable == 'true' || steps.version.outputs.action == 'ready' || steps.version.outputs.action == 'reserved' }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          fetch-depth: 0
+          submodules: true
+          token: ${{ github.event_name == 'push' && secrets.RELEASE_GH_TOKEN || github.token }}
+
+      - name: Keep synchronized stable branches on the released version
+        id: channel
+        if: github.event_name == 'push'
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
+          git fetch --no-tags origin main
+          MAIN_SHA="$(git rev-parse origin/main)"
+          # Stable reconciliation may preserve dev history with a merge commit.
+          # Its tree still matches main; bumping it would undo synchronization
+          # before the stable publisher can advance the Docker :dev tag.
+          if [[ "$GITHUB_REF_NAME" == dev && "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
+             git merge-base --is-ancestor "$MAIN_SHA" "$GITHUB_SHA" &&
+             git diff --quiet "$MAIN_SHA" "$GITHUB_SHA"; then
+            echo 'stable=true' >> "$GITHUB_OUTPUT"
+          else
+            echo 'stable=false' >> "$GITHUB_OUTPUT"
+          fi
+
+      - id: version
+        if: github.event_name == 'push' && steps.channel.outputs.stable != 'true'
+        uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
+        with:
+          package: archivebox
+          branch: ${{ github.ref_name }}
+          tag-prefix: v
+          version-scheme: ${{ github.ref_name == 'main' && 'patch' || 'rc' }}

.github/workflows/release-candidate.yml

@@ -1,65 +0,0 @@
-name: Release candidate
-
-on:
-  push:
-    branches: [dev]
-
-permissions:
-  actions: read
-  contents: write
-  packages: write
-
-concurrency:
-  group: release-candidate-dev
-  cancel-in-progress: true
-
-jobs:
-  prepare:
-    runs-on: ubuntu-24.04
-    outputs:
-      action: ${{ steps.version.outputs.action }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          token: ${{ secrets.RELEASE_GH_TOKEN }}
-
-      - id: version
-        uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
-        with:
-          package: archivebox
-          branch: dev
-          tag-prefix: v
-          version-scheme: rc
-
-  python-artifacts:
-    needs: prepare
-    if: needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved'
-    uses: ./.github/workflows/pip.yml
-    with:
-      full_tests: false
-    secrets: inherit
-
-  docker-digests:
-    needs: prepare
-    if: needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved'
-    uses: ./.github/workflows/docker.yml
-    with:
-      push_digests: true
-      full_tests: false
-    secrets: inherit
-
-  opencode:
-    needs: prepare
-    if: needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved'
-    strategy:
-      fail-fast: false
-      matrix:
-        test_file:
-          - archivebox/tests/test_opencode_agent.py
-          - archivebox/tests/test_opencode_browser.py
-    uses: ./.github/workflows/test-parallel.yml
-    with:
-      test_file: ${{ matrix.test_file }}
-    secrets: inherit

.github/workflows/release.yml

@@ -2,36 +2,56 @@ name: Release
 
 on:
   workflow_run:
-    workflows: [CI, Release candidate]
+    workflows: [CI]
     types: [completed]
     branches: [dev, main]
+  workflow_dispatch:
+    inputs:
+      release_branch:
+        description: Branch being promoted
+        required: true
+        type: choice
+        options: [main, dev]
+      release_sha:
+        description: Exact tested commit to release
+        required: true
+        type: string
+      ci_run_id:
+        description: Successful CI run containing the tested Python artifacts
+        required: true
+        type: string
+      docker_digest_run_id:
+        description: Manual Docker digest build run containing the tested image digests
+        required: true
+        type: string
 
 permissions: {}
 
 concurrency:
-  group: release-${{ github.repository }}-${{ github.event.workflow_run.head_branch }}
+  group: release-${{ github.repository }}-${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
   cancel-in-progress: false
 
 env:
   DOCKERHUB_IMAGE: archivebox/archivebox
   GHCR_IMAGE: ghcr.io/archivebox/archivebox
-  RELEASE_BRANCH: ${{ github.event.workflow_run.head_branch }}
-  RELEASE_SHA: ${{ github.event.workflow_run.head_sha }}
-  CI_RUN_ID: ${{ github.event.workflow_run.id }}
+  RELEASE_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
+  RELEASE_SHA: ${{ github.event_name == 'workflow_dispatch' && inputs.release_sha || github.event.workflow_run.head_sha }}
 
 jobs:
   candidate:
     if: >-
       github.event.workflow_run.conclusion == 'success' &&
       github.event.workflow_run.event == 'push' &&
-      ((github.event.workflow_run.head_branch == 'dev' && github.event.workflow_run.name == 'Release candidate') ||
-       (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.name == 'CI'))
+      (github.event.workflow_run.head_branch == 'dev' || github.event.workflow_run.head_branch == 'main') ||
+      (github.event_name == 'workflow_dispatch' && inputs.release_branch == 'main')
     runs-on: ubuntu-24.04
     permissions:
       actions: read
       contents: read
     outputs:
       ready: ${{ steps.verified.outputs.ready }}
+      artifact_run_id: ${{ steps.verified.outputs.artifact_run_id }}
+      digest_run_id: ${{ steps.verified.outputs.digest_run_id }}
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
@@ -40,49 +60,84 @@ jobs:
       - id: owner
         shell: bash
         run: |
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            echo 'ready=true' >> "$GITHUB_OUTPUT"
+          set -Eeuo pipefail
+          BRANCH_SHA="$(git ls-remote origin "refs/heads/$RELEASE_BRANCH" | cut -f1)"
+          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && "$BRANCH_SHA" != "$RELEASE_SHA" ]]; then
+            echo 'ready=false' >> "$GITHUB_OUTPUT"
             exit 0
           fi
           VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
+          if [[ "$RELEASE_BRANCH" == main ]]; then
+            [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
+              echo "Refusing to publish non-stable version $VERSION as a main release" >&2
+              exit 1
+            }
+            git fetch --no-tags origin main
+            git merge-base --is-ancestor "$RELEASE_SHA" origin/main || {
+              echo "Refusing to publish $RELEASE_SHA: it is not on main" >&2
+              exit 1
+            }
+            TAG_TARGET="$(git ls-remote origin "refs/tags/v$VERSION" | cut -f1)"
+            if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && -n "$TAG_TARGET" && "$TAG_TARGET" != "$RELEASE_SHA" ]]; then
+              echo 'ready=false' >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo 'ready=true' >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          # A stable release fast-forwards dev to main. Only main publishes it;
+          # the resulting dev workflow is not a new release candidate.
+          if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]]; then
+            echo 'ready=false' >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
           TARGET="$(git ls-remote origin "refs/tags/release-candidate/$VERSION" | awk '{print $1}')"
           [[ "$TARGET" == "$RELEASE_SHA" ]] && echo 'ready=true' >> "$GITHUB_OUTPUT" || echo 'ready=false' >> "$GITHUB_OUTPUT"
 
       - name: Require successful full CI for this commit
         id: verified
         if: steps.owner.outputs.ready == 'true'
-        timeout-minutes: 40
         env:
           GH_TOKEN: ${{ github.token }}
+          MANUAL_CI_RUN_ID: ${{ inputs.ci_run_id }}
+          MANUAL_DIGEST_RUN_ID: ${{ inputs.docker_digest_run_id }}
         shell: bash
         run: |
           set -Eeuo pipefail
-          CI_RUN_ID=''
-          for _ in {1..12}; do
-            CI_RUN_ID="$(gh run list \
-              --repo "$GITHUB_REPOSITORY" \
-              --workflow ci.yml \
-              --commit "$RELEASE_SHA" \
-              --event push \
-              --limit 1 \
-              --json databaseId \
-              --jq '.[0].databaseId // empty')"
-            [[ -n "$CI_RUN_ID" ]] && break
-            sleep 5
-          done
-          [[ -n "$CI_RUN_ID" ]]
-          gh run watch "$CI_RUN_ID" --repo "$GITHUB_REPOSITORY" --exit-status
-          echo 'ready=true' >> "$GITHUB_OUTPUT"
+          if [[ "$GITHUB_EVENT_NAME" == workflow_dispatch ]]; then
+            CI_RUN_ID="$MANUAL_CI_RUN_ID"
+            DIGEST_RUN_ID="$MANUAL_DIGEST_RUN_ID"
+            CI_STATUS="$(gh run view "$CI_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status,conclusion,headSha,event,name \
+              --jq '[.status, .conclusion, .headSha, .event, .name] | @tsv')"
+            [[ "$CI_STATUS" == "$(printf 'completed\tsuccess\t%s\tpush\tCI' "$RELEASE_SHA")" ]]
+          else
+            CI_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml \
+              --commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
+              --json databaseId,status,conclusion \
+              --jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
+            [[ -n "$CI_RUN_ID" ]] || exit 0
+            DIGEST_RUN_ID="$CI_RUN_ID"
+          fi
+          ARTIFACT_RUN_ID="$CI_RUN_ID"
+          {
+            echo 'ready=true'
+            echo "artifact_run_id=$ARTIFACT_RUN_ID"
+            echo "digest_run_id=$DIGEST_RUN_ID"
+          } >> "$GITHUB_OUTPUT"
 
   python-release:
     name: Publish tested Python artifacts
     needs: candidate
     if: needs.candidate.outputs.ready == 'true'
     runs-on: ubuntu-24.04
+    env:
+      CI_RUN_ID: ${{ needs.candidate.outputs.artifact_run_id }}
+      DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
     environment: pypi
     permissions:
       actions: read
       contents: write
+      deployments: read
       id-token: write
     outputs:
       version: ${{ steps.version.outputs.version }}
@@ -106,6 +161,26 @@ jobs:
           version: "0.11.3"
           enable-cache: false
 
+      # This precedes every stable upload, including workflow_dispatch. A green
+      # build is not evidence that our automatically updated test servers ran it.
+      - name: Download tested Docker base-image metadata
+        if: env.RELEASE_BRANCH == 'main'
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          path: ${{ runner.temp }}/tested-docker-artifacts
+          pattern: digest-*
+          merge-multiple: true
+          github-token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
+
+      - name: Require matching acceptance on Cabbage
+        if: env.RELEASE_BRANCH == 'main'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          STAGING_DOCKER_ARTIFACT_DIR: ${{ runner.temp }}/tested-docker-artifacts
+        run: uv run --no-project python bin/verify-staging-release.py
+
       - name: Resolve release binaries through abxpkg
         env:
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
@@ -154,6 +229,10 @@ jobs:
           VERSION='${{ steps.version.outputs.version }}'
           case "$RELEASE_BRANCH" in
             main)
+              [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
+                echo "Refusing to publish non-stable version $VERSION from main" >&2
+                exit 1
+              }
               ;;
             dev)
               [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]] || {
@@ -191,20 +270,92 @@ jobs:
   docker-release:
     name: Publish tested multiarch image
     if: >-
-      github.event.workflow_run.conclusion == 'success' &&
-      github.event.workflow_run.event == 'push'
-    needs: python-release
+      (github.event_name == 'workflow_dispatch') ||
+      (github.event.workflow_run.conclusion == 'success' &&
+       github.event.workflow_run.event == 'push')
+    needs: [candidate, python-release]
     runs-on: ubuntu-24.04
+    env:
+      DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
     permissions:
       actions: read
-      contents: read
+      contents: write
       packages: write
     steps:
       - name: Checkout the tested commit
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
-          fetch-depth: 1
+          fetch-depth: 0
           ref: ${{ env.RELEASE_SHA }}
+          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
+
+      - name: Reconcile dev to the tested stable release
+        if: env.RELEASE_BRANCH == 'main' && needs.python-release.outputs.release_ready == 'true'
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          git fetch --no-tags origin main dev
+          MAIN_SHA="$(git rev-parse origin/main)"
+          DEV_SHA="$(git rev-parse origin/dev)"
+          if [[ "$MAIN_SHA" != "$RELEASE_SHA" || "$DEV_SHA" == "$RELEASE_SHA" ]]; then
+            exit 0
+          fi
+          push_dev() {
+            if ! git push origin "$1:refs/heads/dev"; then
+              CURRENT_DEV="$(git ls-remote origin refs/heads/dev | cut -f1)"
+              if [[ "$CURRENT_DEV" == "$DEV_SHA" ]]; then
+                echo "Failed to push dev although it still points to $DEV_SHA" >&2
+                exit 1
+              fi
+              echo "Leaving dev at $CURRENT_DEV: it advanced during release reconciliation"
+            fi
+          }
+          if git merge-base --is-ancestor "$DEV_SHA" "$RELEASE_SHA"; then
+            push_dev "$RELEASE_SHA"
+            exit 0
+          else
+            STATUS=$?
+            [[ "$STATUS" -eq 1 ]] || exit "$STATUS"
+          fi
+          BASE_SHA="$(git merge-base "$MAIN_SHA" "$DEV_SHA")"
+          if git diff --quiet "$BASE_SHA" "$DEV_SHA" -- . ':!pyproject.toml' ':!uv.lock' ':!etc/package.json'; then
+            :
+          else
+            STATUS=$?
+            [[ "$STATUS" -eq 1 ]] || exit "$STATUS"
+            echo "Leaving dev at $DEV_SHA: it contains changes beyond release metadata"
+            exit 0
+          fi
+          normalized_metadata() {
+            case "$2" in
+              pyproject.toml)
+                git show "$1:$2" | awk '
+                  /^\[/ {section = $0}
+                  section == "[project]" && /^version = "[^"]+"$/ {print "version = \"__VERSION__\""; next}
+                  section == "[tool.bumpver]" && /^current_version = "[^"]+"$/ {print "current_version = \"__VERSION__\""; next}
+                  {print}
+                ' ;;
+              uv.lock)
+                git show "$1:$2" | awk 'previous == "name = \"archivebox\"" && /^version = / {print "version = \"__VERSION__\""; previous = $0; next} {print; previous = $0}' ;;
+              etc/package.json)
+                git show "$1:$2" | sed -E 's/^(  "version": )"[^"]+"(,?)$/\1"__VERSION__"\2/' ;;
+            esac
+          }
+          for path in pyproject.toml uv.lock etc/package.json; do
+            BASE_METADATA="$(normalized_metadata "$BASE_SHA" "$path")"
+            DEV_METADATA="$(normalized_metadata "$DEV_SHA" "$path")"
+            if [[ "$BASE_METADATA" != "$DEV_METADATA" ]]; then
+              echo "Leaving dev at $DEV_SHA: $path contains changes beyond release version metadata"
+              exit 0
+            fi
+          done
+          # A concurrent release bot may bump only dev's own version metadata.
+          # Keep that commit as a parent while preserving the exact tested stable tree.
+          git switch --detach "$MAIN_SHA"
+          git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \
+            merge -s ours --no-ff -m "Merge tested stable release into dev" "$DEV_SHA"
+          MERGE_SHA="$(git rev-parse HEAD)"
+          push_dev "$MERGE_SHA"
 
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
@@ -227,20 +378,22 @@ jobs:
             --lib="$ABXPKG_LIB_DIR" \
             --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
             --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null
-          for name in uv git jq docker; do
+          for name in uv gh git jq docker; do
             test -L "$ABXPKG_LIB_DIR/env/bin/$name"
             test -x "$ABXPKG_LIB_DIR/env/bin/$name"
           done
           {
             echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
             echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
+            echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
             echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
             echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
           } >> "$GITHUB_ENV"
 
       - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
         with:
-          install: true
+          # These images were already built and tested; only publish manifests.
+          driver: docker
 
       - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
         with:
@@ -261,16 +414,31 @@ jobs:
           merge-multiple: true
           github-token: ${{ github.token }}
           repository: ${{ github.repository }}
-          run-id: ${{ env.CI_RUN_ID }}
+          run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
+
+      - name: Verify exact successful digest build run
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -Eeuo pipefail
+          RUN_STATUS="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status --jq '.status')"
+          RUN_CONCLUSION="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json conclusion --jq '.conclusion')"
+          [[ "$RUN_STATUS" == completed && "$RUN_CONCLUSION" == success ]]
+          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
+            RUN_SHA="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json headSha --jq '.headSha')"
+            [[ "$RUN_SHA" == "$RELEASE_SHA" ]]
+          fi
 
       - name: Verify digest set and collect tags
         id: docker_meta
         shell: bash
         run: |
           set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
           [[ "${#DIGESTS[@]}" -eq 2 ]]
           for digest in "${DIGESTS[@]}"; do [[ "$digest" =~ ^[0-9a-f]{64}$ ]]; done
+          [[ "$(< /tmp/digests/source-ref)" == "$RELEASE_SHA" ]]
 
           VERSION='${{ needs.python-release.outputs.version }}'
           SHORT_SHA="${RELEASE_SHA::12}"
@@ -280,12 +448,46 @@ jobs:
             TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")"
             TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
           fi
+          if [[ "$RELEASE_BRANCH" == main ]]; then
+            [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
+              echo "Refusing to tag :latest with prerelease $VERSION" >&2
+              exit 1
+            }
+            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] || {
+              echo "Refusing to tag :latest: v$VERSION does not point to $RELEASE_SHA" >&2
+              exit 1
+            }
+            $GIT_BINARY fetch --no-tags origin main
+            $GIT_BINARY merge-base --is-ancestor "$RELEASE_SHA" origin/main || {
+              echo "Refusing to tag :latest: $RELEASE_SHA is not on main" >&2
+              exit 1
+            }
+            MAIN_VERSION="$($GIT_BINARY show origin/main:pyproject.toml | sed -nE 's/^version = "([^"]+)".*/\1/p' | head -n 1)"
+            [[ "$MAIN_VERSION" == "$VERSION" ]] || {
+              echo "Refusing to tag :latest: main is now $MAIN_VERSION, not $VERSION" >&2
+              exit 1
+            }
+          fi
+          SYNC_DEV=false
+          if [[ "$RELEASE_BRANCH" == main ]]; then
+            $GIT_BINARY fetch --no-tags origin main dev
+            MAIN_TARGET="$($GIT_BINARY rev-parse origin/main)"
+            DEV_TARGET="$($GIT_BINARY rev-parse origin/dev)"
+            # Recovery can add CI/docs fixes after the immutable release tag.
+            # Both branches and every shipped application file must agree.
+            if $GIT_BINARY merge-base --is-ancestor "$MAIN_TARGET" "$DEV_TARGET" &&
+               $GIT_BINARY diff --quiet "$MAIN_TARGET" "$DEV_TARGET" &&
+               $GIT_BINARY diff --quiet "$RELEASE_SHA" "$MAIN_TARGET" -- . ':!.github' ':!README.md'; then
+              SYNC_DEV=true
+            fi
+          fi
           {
             echo 'dockerhub_tags<<EOF'
             if [[ "$RELEASE_BRANCH" == dev ]]; then
               echo "${DOCKERHUB_IMAGE}:dev"
             elif [[ "$RELEASE_BRANCH" == main ]]; then
               echo "${DOCKERHUB_IMAGE}:latest"
+              [[ "$SYNC_DEV" == true ]] && echo "${DOCKERHUB_IMAGE}:dev"
             fi
             echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
             [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${DOCKERHUB_IMAGE}:${VERSION}"
@@ -295,6 +497,7 @@ jobs:
               echo "${GHCR_IMAGE}:dev"
             elif [[ "$RELEASE_BRANCH" == main ]]; then
               echo "${GHCR_IMAGE}:latest"
+              [[ "$SYNC_DEV" == true ]] && echo "${GHCR_IMAGE}:dev"
             fi
             echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
             [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${GHCR_IMAGE}:${VERSION}"
@@ -305,7 +508,7 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
           TAG_ARGS=()
           while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
           REFS=()
@@ -316,24 +519,54 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
           TAG_ARGS=()
           while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
           REFS=()
           for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
           $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
 
-      - name: Verify published Docker images run
+      - name: Verify published tags contain the exact tested images
+        env:
+          DOCKERHUB_TAGS: ${{ steps.docker_meta.outputs.dockerhub_tags }}
+          GHCR_TAGS: ${{ steps.docker_meta.outputs.ghcr_tags }}
         shell: bash
         run: |
           set -Eeuo pipefail
-          VERSION='${{ needs.python-release.outputs.version }}'
-          SHORT_SHA="${RELEASE_SHA::12}"
-          for image in "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"; do
-            "$DOCKER_BINARY" buildx imagetools inspect "$image" >/dev/null
-            INSTALLED_VERSION="$("$DOCKER_BINARY" run --rm "$image" --version)"
-            test "$INSTALLED_VERSION" = "$VERSION"
-          done
+          # Both architectures already passed CLI/version/capture acceptance by
+          # digest. Verify promotion preserves those exact images and attestations
+          # instead of downloading and running the same image again on one arch.
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
+          EXPECTED="$("$DOCKER_BINARY" buildx imagetools create --dry-run "${REFS[@]}" | "$JQ_BINARY" -ceS '.manifests | sort_by(.digest)')"
+          "$JQ_BINARY" -e 'map(select(.platform.os == "linux") | .platform.architecture) | sort == ["amd64", "arm64"]' <<< "$EXPECTED"
+          while IFS= read -r image; do
+            [[ -n "$image" ]] || continue
+            ACTUAL="$("$DOCKER_BINARY" buildx imagetools inspect --raw "$image" | "$JQ_BINARY" -ceS 'select(.schemaVersion == 2) | .manifests | sort_by(.digest)')"
+            [[ "$ACTUAL" == "$EXPECTED" ]] || { echo "Published tag differs from tested images: $image" >&2; exit 1; }
+            echo "Verified exact tested images: $image"
+          done < <(printf '%s\n' "$DOCKERHUB_TAGS" "$GHCR_TAGS")
+
+      - name: Record published image source
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          # workflow_run can report the default branch's head even when this
+          # run published a different branch/SHA. Preserve the verified target.
+          [[ "$RELEASE_BRANCH" == dev || "$RELEASE_BRANCH" == main ]]
+          [[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          mkdir -p "$RUNNER_TEMP/published-image-source"
+          # shellcheck disable=SC2016
+          "$JQ_BINARY" -n --arg branch "$RELEASE_BRANCH" --arg sha "$RELEASE_SHA" \
+            '{branch: $branch, sha: $sha}' > "$RUNNER_TEMP/published-image-source/source.json"
+
+      - name: Upload published image source
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: published-image-source
+          path: ${{ runner.temp }}/published-image-source/source.json
+          if-no-files-found: error
 
       - name: Update Docker Hub README
         uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4
@@ -346,6 +579,7 @@ jobs:
     name: Cascade verified release
     if: needs.python-release.outputs.release_ready == 'true'
     needs:
+      - candidate
       - python-release
       - docker-release
     permissions:
@@ -356,5 +590,6 @@ jobs:
       upstream_package: archivebox
       upstream_version: ${{ needs.python-release.outputs.version }}
       upstream_artifact_name: python-distributions
+      upstream_run_id: ${{ needs.candidate.outputs.artifact_run_id }}
     secrets:
       release_token: ${{ secrets.RELEASE_GH_TOKEN }}

.github/workflows/screenshots.yml

@@ -1,29 +1,31 @@
 name: Capture publicsite screenshots
 
 on:
-  push:
-    branches:
-      - dev
-    paths:
-      - archivebox/**
-      - bin/collect_ui_screenshots.*
-      - bin/generate_ui_screenshot_gallery.py
-      - bin/take_screenshot.js
-      - docs/Screenshots.md
-      - pyproject.toml
-      - uv.lock
-      - .github/workflows/screenshots.yml
+  schedule:
+    - cron: '37 5,17 * * *'
   workflow_dispatch:
 
 permissions:
   contents: read
+  actions: read
 
 concurrency:
-  group: ${{ github.event_name == 'workflow_dispatch' && format('publicsite-captures-manual-{0}', github.run_id) || 'publicsite-captures' }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: false
 
 jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
   deploy:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout
@@ -69,3 +71,11 @@ jobs:
           path: publicsite/screenshots
           if-no-files-found: error
           retention-days: 90
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, deploy]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled work completed successfully.'

.github/workflows/staging-acceptance.yml

@@ -0,0 +1,117 @@
+name: Staging acceptance
+
+on:
+  workflow_run:
+    workflows: [Release]
+    types: [completed]
+
+permissions: {}
+
+concurrency:
+  group: staging-acceptance
+  cancel-in-progress: false
+
+jobs:
+  published:
+    if: github.event.workflow_run.conclusion == 'success'
+    runs-on: ubuntu-24.04
+    permissions:
+      actions: read
+    outputs:
+      ready: ${{ steps.source.outputs.ready }}
+      sha: ${{ steps.source.outputs.sha }}
+    steps:
+      - id: image
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_RUN_ID: ${{ github.event.workflow_run.id }}
+        run: |
+          set -Eeuo pipefail
+          published=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RELEASE_RUN_ID/jobs?per_page=100" --jq 'any(.jobs[]; .name == "Publish tested multiarch image" and .conclusion == "success")')
+          echo "published=$published" >> "$GITHUB_OUTPUT"
+      - name: Download published image source
+        if: steps.image.outputs.published == 'true'
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: published-image-source
+          path: ${{ runner.temp }}/published-image-source
+          github-token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          run-id: ${{ github.event.workflow_run.id }}
+      - name: Select published dev image
+        id: source
+        if: steps.image.outputs.published == 'true'
+        run: |
+          set -Eeuo pipefail
+          # The triggering run may report the default branch as its head;
+          # this artifact names the image source verified by the publisher.
+          source="$RUNNER_TEMP/published-image-source/source.json"
+          jq -e 'type == "object" and (keys | sort) == ["branch", "sha"]' "$source" >/dev/null
+          branch=$(jq -er '.branch | select(type == "string")' "$source")
+          sha=$(jq -er '.sha | select(type == "string")' "$source")
+          [[ "$branch" == dev || "$branch" == main ]]
+          [[ "$sha" =~ ^[0-9a-f]{40}$ ]]
+          echo "sha=$sha" >> "$GITHUB_OUTPUT"
+          if [[ "$branch" == dev ]]; then
+            echo 'ready=true' >> "$GITHUB_OUTPUT"
+          else
+            echo 'ready=false' >> "$GITHUB_OUTPUT"
+          fi
+  acceptance:
+    # The privileged key is never exposed to PRs or arbitrary dispatch refs.
+    needs: published
+    if: needs.published.outputs.ready == 'true'
+    runs-on: [self-hosted, Linux, X64, ugnas-linux]
+    environment: staging
+    timeout-minutes: 110
+    permissions:
+      contents: read
+      deployments: write
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [cabbage]
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          ref: ${{ needs.published.outputs.sha }}
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e
+        with:
+          version: '0.11.3'
+      - name: Prepare browser and restricted staging identity
+        env:
+          STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
+          STAGING_KNOWN_HOSTS: ${{ secrets.STAGING_KNOWN_HOSTS }}
+          ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/staging-browser
+        run: |
+          set -Eeuo pipefail
+          umask 077
+          printf '%s\n' "$STAGING_SSH_KEY" > "$RUNNER_TEMP/staging-key"
+          printf '%s\n' "$STAGING_KNOWN_HOSTS" > "$RUNNER_TEMP/staging-known-hosts"
+          # Use the candidate's normal provider/cache path, not a second
+          # Playwright-managed browser that bypasses abxpkg validation.
+          ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
+          CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
+          uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
+          CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
+          test -x "$CHROME_BINARY"
+          echo "CHROME_BINARY=$CHROME_BINARY" >> "$GITHUB_ENV"
+      - name: Verify actual deployment, full captures and rendered snapshots
+        env:
+          GH_TOKEN: ${{ github.token }}
+          STAGING_TARGET: ${{ matrix.target }}
+          STAGING_SHA: ${{ needs.published.outputs.sha }}
+          STAGING_KEY_FILE: ${{ runner.temp }}/staging-key
+          STAGING_KNOWN_HOSTS_FILE: ${{ runner.temp }}/staging-known-hosts
+        run: uv run --no-project --with playwright==1.63.0 python bin/check-staging-deployment.py
+      - name: Remove staging identity
+        if: always()
+        run: rm -f "$RUNNER_TEMP/staging-key" "$RUNNER_TEMP/staging-known-hosts"
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        if: always()
+        with:
+          name: staging-${{ matrix.target }}-${{ needs.published.outputs.sha }}
+          path: staging-evidence/
+          if-no-files-found: error

.github/workflows/test-parallel.yml

@@ -1,6 +1,13 @@
 name: Parallel Tests
 
 on:
+  workflow_dispatch:
+    inputs:
+      test_file:
+        description: Run one discovered test file
+        required: true
+        default: archivebox/tests/test_cli_list.py
+        type: string
   workflow_call:
     inputs:
       test_file:
@@ -19,11 +26,20 @@ env:
 jobs:
   discover-tests:
     name: Discover every test
+    # Selection must run on GitHub so an offline NAS cannot block the fallback.
     runs-on: ubuntu-24.04
     outputs:
       tests: ${{ steps.discover.outputs.tests }}
+      runner: ${{ steps.runner.outputs.use-runner || '["ubuntu-24.04"]' }}
 
     steps:
+      - name: Select available CI capacity
+        id: runner
+        if: vars.UGNAS_CI_ENABLED == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
+        uses: ArchiveBox/monorepo/.github/actions/select-ci-runner@main
+        with:
+          github-token: ${{ secrets.RUNNER_STATUS_TOKEN }}
+
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           submodules: true
@@ -66,6 +82,7 @@ jobs:
         id: discover
         env:
           TEST_FILE: ${{ inputs.test_file }}
+          UGNAS_CI_MAX_JOBS: ${{ vars.UGNAS_CI_MAX_JOBS || 3 }}
         run: |
           set -Eeuo pipefail
           mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
@@ -80,13 +97,13 @@ jobs:
   tests:
     name: ${{ matrix.test.name }}
     needs: discover-tests
-    runs-on: ubuntu-24.04
+    # Opt in trusted branches only; PRs always use GitHub-hosted runners.
+    # Ordinary Linux tests are eligible; # ci-runner: hosted declares exceptions.
+    # The hosted discovery job falls back when workers are offline or the
+    # availability API fails. UGNAS_CI_ENABLED=false disables the pool entirely.
+    runs-on: ${{ fromJSON(matrix.test.ugnas && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
     env:
-      ABXPKG_LIB_DIR: /tmp/archivebox-lib
       CHROME_HEADLESS: "true"
-      DATA_DIR: /tmp/archivebox-test-data
-      PERSONAS_DIR: /tmp/abx-personas
-      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
 
     strategy:
       fail-fast: false
@@ -94,6 +111,28 @@ jobs:
         test: ${{ fromJson(needs.discover-tests.outputs.tests) }}
 
     steps:
+      - name: Set job-local runtime paths
+        env:
+          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
+        run: |
+          workers=2
+          cache_suffix=''
+          if [[ "$CI_RUNNER_ENVIRONMENT" == github-hosted ]]; then
+            workers="$(nproc)"
+            cache_suffix="-workers-$workers"
+            # Avoid the hosted image's cold Chromium executable page faults.
+            # abxpkg owns installation and reuse in the existing runtime cache.
+            echo "CHROME_BINPROVIDERS=playwright,puppeteer" >> "$GITHUB_ENV"
+          fi
+          {
+            echo "CI_TEST_WORKERS=$workers"
+            echo "ABXPKG_CACHE_WORKERS_SUFFIX=$cache_suffix"
+            echo "ABXPKG_LIB_DIR=$RUNNER_TEMP/archivebox-lib"
+            echo "DATA_DIR=$RUNNER_TEMP/archivebox-test-data"
+            echo "PERSONAS_DIR=$RUNNER_TEMP/abx-personas"
+            echo "CHROME_USER_DATA_DIR=$RUNNER_TEMP/abx-personas/Default/chrome_profile"
+          } >> "$GITHUB_ENV"
+
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           submodules: true
@@ -103,6 +142,7 @@ jobs:
         with:
           python-version: ${{ env.CI_PYTHON_VERSION }}
           architecture: x64
+        id: python
 
       - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
@@ -110,6 +150,41 @@ jobs:
           enable-cache: false
           cache-dependency-glob: uv.lock
 
+      # Self-hosted tool storage survives jobs. Normal abxpkg validation and
+      # install locks allow reuse across files; cold-install tests use tmp_path.
+      - name: Reuse persistent runner tools
+        if: runner.environment == 'self-hosted'
+        env:
+          PERSISTENT_ABXPKG_LIB: ${{ runner.tool_cache }}/abxpkg/${{ github.repository }}/${{ steps.python.outputs.python-version }}
+        run: |
+          mkdir -p "$PERSISTENT_ABXPKG_LIB"
+          echo "ABXPKG_LIB_DIR=$PERSISTENT_ABXPKG_LIB" >> "$GITHUB_ENV"
+
+      # Restored binaries still pass normal abxpkg resolution/version checks.
+      # Installation and isolation tests explicitly choose fresh temporary libs.
+      - name: Identify runtime dependency cache
+        shell: bash
+        run: |
+          uv run --no-project python - <<'PYTHON' >> "$GITHUB_ENV"
+          import hashlib
+          import json
+          import tomllib
+          from pathlib import Path
+
+          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
+          versions = {p["name"]: p["version"] for p in packages if p["name"] in {"abxpkg", "abx-plugins"} and "editable" not in p["source"]}
+          print("ABXPKG_CACHE_DEPS=" + hashlib.sha256(json.dumps(versions, sort_keys=True).encode()).hexdigest())
+          PYTHON
+
+      - name: Reuse validated runtime dependencies
+        if: runner.environment != 'self-hosted'
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ${{ runner.temp }}/archivebox-lib
+          key: abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-${{ env.ABXPKG_CACHE_DEPS }}${{ env.ABXPKG_CACHE_WORKERS_SUFFIX }}
+          restore-keys: |
+            abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-
+
       - name: Resolve optional build dependencies through abxpkg
         if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
         env:
@@ -143,23 +218,38 @@ jobs:
             extra_args=(--extra "$TEST_EXTRA")
           fi
           uv sync --no-cache --locked --dev "${extra_args[@]}"
-
       - name: Run ${{ matrix.test.name }}
         env:
           TEST_PATHS: ${{ matrix.test.paths_arg }}
+          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
         run: |
           set -Eeuo pipefail
+          echo "Running pytest with $CI_TEST_WORKERS workers on $CI_RUNNER_ENVIRONMENT"
           read -r -a test_paths <<< "$TEST_PATHS"
+          test_args=()
           for test_path in "${test_paths[@]}"; do
             test -f "$test_path"
+            test_args+=("$GITHUB_WORKSPACE/$test_path")
           done
-          for test_path in "${test_paths[@]}"; do
-            test_name="$(basename "$test_path" .py)"
-            test_slug="${test_name//[^A-Za-z0-9_.-]/_}"
-            export DATA_DIR="$RUNNER_TEMP/archivebox-test-data/$test_slug"
-            export PERSONAS_DIR="$RUNNER_TEMP/abx-personas/$test_slug"
-            export CHROME_USER_DATA_DIR="$PERSONAS_DIR/Default/chrome_profile"
-            mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest/$test_slug"
-            uv run --no-cache --no-sync --no-sources pytest -vs "$test_path" \
-              --basetemp="$RUNNER_TEMP/pytest/$test_slug"
-          done
+          # Each test still gets isolated DB/filesystem state from conftest.
+          # Keep one worker pool per batch instead of starting it for every file.
+          mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest"
+          uv run --no-cache --no-sync --no-sources pytest -vs -n "$CI_TEST_WORKERS" "${test_args[@]}" --durations=20 \
+            --basetemp="$RUNNER_TEMP/pytest" \
+            --junitxml="$RUNNER_TEMP/test-results/results.xml"
+      - name: Save test results
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: test-results-${{ strategy.job-index }}
+          path: ${{ runner.temp }}/test-results/*.xml
+          if-no-files-found: error
+          retention-days: 14
+      - name: Save failed runtime diagnostics
+        if: failure()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: runtime-logs-${{ strategy.job-index }}
+          path: ${{ runner.temp }}/pytest/**/*.log
+          if-no-files-found: ignore
+          retention-days: 14

.github/workflows/test.yml

@@ -2,6 +2,12 @@ name: Install and CLI Compatibility
 
 on:
   workflow_call:
+    inputs:
+      platform:
+        description: Platform for install and CLI compatibility
+        required: false
+        default: linux
+        type: string
 
 env:
   DOCKER_IMAGE: archivebox-ci
@@ -19,16 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-24.04
-            os_name: linux
-            python: "3.14"
-          - os: macos-15
-            os_name: macOS
-            python: "3.13"
-          - os: macos-15
-            os_name: macOS
-            python: "3.14"
+        include: ${{ fromJSON(inputs.platform == 'macos' && '[{"os":"macos-15","os_name":"macOS","python":"3.13"},{"os":"macos-15","os_name":"macOS","python":"3.14"}]' || '[{"os":"ubuntu-24.04","os_name":"linux","python":"3.14"}]') }}
 
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

.pre-commit-config.yaml

@@ -56,6 +56,8 @@ repos:
       - id: mixed-line-ending
       - id: fix-byte-order-marker
       - id: end-of-file-fixer
+        # Preserve the exact bytes of captured cryptographic proof fixtures.
+        exclude: ^archivebox/tests/fixtures/legacy_proofs/.*\.json$
       - id: detect-private-key
       - id: debug-statements
       - id: forbid-submodules

AGENTS.md

@@ -77,11 +77,7 @@ archivebox run
 
 ## Verification
 
-Use targeted tests for focused work:
+Run the test suite with `uv run pytest archivebox/tests -x` and repository checks
+with `uv run prek run --all-files`. For focused work, select the relevant tests.
 
-```bash
-uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
-uv run prek run --all-files
-```
-
-Releases are published only by `.github/workflows/release.yml` after the complete `dev` CI workflow succeeds. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.
+Releases are published only by `.github/workflows/release.yml` after complete CI succeeds for the exact commit. Pushes to `main` prepare stable patch releases automatically; `dev` prepares release candidates. Version selection and dependency repinning are owned by CI and the monorepo coordinator. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.

Dockerfile

@@ -3,14 +3,14 @@
 # Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
 # WHY: abx-dl owns Python, Node, Chromium, and extraction dependencies so the
 # standalone downloader stays independent of ArchiveBox's server features.
-# ArchiveBox-only dependencies (Sonic, OpenCode, supervisor, Django) belong in
+# ArchiveBox-only dependencies (server plugins, supervisor, Django) belong in
 # the layers added by this Dockerfile, never in the shared abx-dl base image.
 # REQUIRED INSTALLATION CONTRACT — do not work around failures:
 # 1. IMAGE CONSTRUCTION: preinstall ALL plugins supported by runtime="archivebox"
 #    and ALL their required_binaries, regardless of default enabled state.
 #    Inherit the complete downloader dependencies from the tested abx-dl image;
 #    install every remaining ArchiveBox-only dependency in this image's layers,
-#    including OpenCode and Sonic. Never put server dependencies in abx-dl.
+#    including optional server plugins. Never put server dependencies in abx-dl.
 #    Use the normal plugin dependency pipeline: install command -> config.json
 #    required_binaries -> abxpkg. Never add manual apt/pip/npm installs to hide
 #    failures in that pipeline. OS/interpreter bootstrap is a separate concern,
@@ -43,7 +43,7 @@
 #       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:dev \
 #       -t archivebox:multistage
 
-ARG ABX_DL_IMAGE=archivebox/abx-dl:dev
+ARG ABX_DL_IMAGE=archivebox/abx-dl:main
 
 FROM archivebox/sonic:1.4.9 AS sonic
 FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
@@ -125,6 +125,7 @@ FROM archivebox-runtime-base AS archivebox-builder
 WORKDIR "$CODE_DIR"
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
     --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    --mount=type=bind,source=uv.lock,target=/app/uv.lock \
     <<'EOF'
 echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
 export UV_NO_CACHE=true
@@ -137,48 +138,14 @@ apt-get install -qq -y --no-install-recommends x11-utils
 # Do not rewrite inherited native libraries merely to strip them again.
 /usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print > /tmp/archivebox-inherited-libraries
 
-mkdir -p /tmp/archivebox-uv-project
-/venv/bin/python - <<'PY'
-from pathlib import Path
-import json
-import re
-import urllib.request
-
-source = Path("/app/pyproject.toml")
-target = Path("/tmp/archivebox-uv-project/pyproject.toml")
-text = source.read_text()
-text = text.replace(
-    'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
-    'environments = ["sys_platform == \'linux\'"]',
-)
-
-# Docker builds need the just-published internal abx wheels immediately, but
-# PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a
-# Docker-only dependency view from the version JSON so the published package
-# metadata stays normal while image builds remain resumable after a release.
-for package in ("abxbus", "abxpkg", "abx-plugins", "abx-dl"):
-    match = re.search(
-        rf'"{re.escape(package)}(?P<operator>==|>=)(?P<version>[^"]+)"',
-        text,
-    )
-    if not match:
-        continue
-    version = match.group("version")
-    with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
-        data = json.load(response)
-    wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
-    text = re.sub(
-        rf'"{re.escape(package)}(?:==|>=)[^"]+"',
-        f'"{package} @ {wheel_url}"',
-        text,
-        count=1,
-    )
-
-target.write_text(text)
-PY
-
+# The old Docker-only dependency rewrite predated this checkout's lockfile:
+# it fetched version JSON to avoid stale PyPI Simple listings. That extra API
+# can itself lag or return 503s. The verified lock already records exact public
+# artifact URLs and hashes, including extras and platform markers. Consume it
+# directly, retaining the freshness check instead of resolving a second graph.
 /usr/bin/uv sync \
-    --project /tmp/archivebox-uv-project \
+    --project /app \
+    --locked \
     --no-cache \
     --no-dev \
     --inexact \
@@ -189,6 +156,22 @@ PY
     --no-install-package abx-plugins \
     --no-install-package abx-dl \
     --no-sources
+# Sync may replace inherited dependencies (e.g. pydantic/rich-click), removing
+# their bytecode. Runtime writes are disabled, so missing caches would make
+# every hook compile them again. Compile only missing entries: compileall's
+# timestamp fast path rewrites even valid checked-hash caches, copying the
+# entire inherited venv into this OCI layer for no runtime benefit.
+/usr/bin/uv run --no-project --no-sync python - <<'PY'
+import importlib.util
+from pathlib import Path
+import py_compile
+import sysconfig
+
+for source in Path(sysconfig.get_path("purelib")).rglob("*.py"):
+    cache = Path(importlib.util.cache_from_source(str(source)))
+    if not cache.is_file():
+        py_compile.compile(str(source), doraise=True, invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH)
+PY
 builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
 ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
 /usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
@@ -209,7 +192,7 @@ rm -rf "$builder_abxpkg_lib_dir"
 rm -rf /venv/lib/python3.*/site-packages/pip* \
     /venv/lib/python3.*/site-packages/wheel* \
     /venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel
-rm -rf /var/lib/apt/lists/* /tmp/archivebox-uv-project
+rm -rf /var/lib/apt/lists/*
 EOF
 
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
@@ -283,18 +266,22 @@ RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777
 
 # Install the complete ArchiveBox catalog, including disabled server plugins.
 # Explicit plugin names make installation independent of enabled defaults.
-RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-opencode-$TARGETARCH \
+RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-server-plugins-$TARGETARCH \
     chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" /var/tmp/abxpkg-cache \
     && chmod 1777 /var/tmp/abxpkg-cache \
     && export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
     && env ABXPKG_TMP_CACHE_DIR=/var/tmp/abxpkg-cache XDG_CACHE_HOME=/var/tmp/abxpkg-cache \
         setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install $ABX_DOCKER_PLUGINS \
-    # pnpm includes musl variants that Debian's glibc runtime cannot use.
-    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules" -type l -name 'opencode-linux-*-musl' -delete \
-    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.pnpm" -maxdepth 1 -type d -name 'opencode-linux-*-musl@*' -exec rm -rf {} +
+    && UV_NO_CACHE=true /usr/bin/uv run --no-project python -c 'from abx_plugins.plugins.opencode.image import prune_incompatible_image_files; prune_incompatible_image_files()' \
+    # pnpm's explicit store lives under ABXPKG_LIB_DIR, regardless of XDG_CACHE_HOME.
+    # Retaining its download copies made image size depend on whether the build
+    # preserved hardlinks to installed files. Remove the store in this layer;
+    # installed files retain their contents, and the offline check below proves
+    # the complete catalog still works without the disposable package payloads.
+    && rm -rf "$ABXPKG_LIB_DIR/cache/pnpm"
 RUN --network=none export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups \
-    bash -c '"$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version && abx-dl install $ABX_DOCKER_PLUGINS'
+    bash -c 'uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_installed; verify_installed()" && abx-dl install $ABX_DOCKER_PLUGINS'
 
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
@@ -314,6 +301,9 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \
     && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
 
 RUN for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
+    && test -f "$(/venv/bin/python -c 'import pydantic; print(pydantic.__cached__)')" \
+    && test -f "$(/venv/bin/python -c 'import rich_click; print(rich_click.__cached__)')" \
+    && test -f "$(/venv/bin/python -c 'import platformdirs; print(platformdirs.__cached__)')" \
     && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \

README.md

@@ -105,12 +105,11 @@ curl -fsSL 'https://get.archivebox.io' | bash
 </details>
 <br/>
 
-
 <div align="center" style="text-align: center">
 <br/><br/>
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic"> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/b2765a33-0d1e-4019-a1db-920c7e00e20e" height="75px" alt="logo" align="top"/> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic">
 <br/><br/>
-<small><a href="https://demo.archivebox.io">Demo</a> | <a href="#screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
+<small><a href="https://demo.archivebox.io">Demo</a> | <a href="https://archivebox.io/screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
 <br/>
 <sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
 <br/><br/>
@@ -124,13 +123,13 @@ curl -fsSL 'https://get.archivebox.io' | bash
 ## Key Features
 
 - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE), own your own data & maintain your privacy by self-hosting
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
 - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
-- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage), [Python API](https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
+- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archivedotorg_enabled) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archivedotorg_enabled) for local-only mode)
 - Support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
 - Support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [infiniscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
 
@@ -221,7 +220,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <ol>
 <li>Run the shortcut for the <code>uv</code> install method. It installs <code>uv</code> first when needed.
 <pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | bash</code></pre>
-<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-automatic-setup-script">Install: Bare Metal</a> in the Wiki. ➡️</i>
+<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-uv-install-shortcut">Install: Bare Metal</a> in the Wiki. ➡️</i>
 </li>
 </ol>
 
@@ -381,7 +380,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 > *Warning: These are contributed by external volunteers and may lag behind the official Docker and `uv` channels.*
 
 <ul>
-<li><s>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></s> (<a href="https://truecharts.org/news/scale-deprecation/">TrueCharts is discontinued</a>, wait for <a href="https://forums.truenas.com/t/the-future-of-electric-eel-and-apps/5409/">Electric Eel</a>)</li>
+<li><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a></li>
 <li><a href="https://unraid.net/community/apps?q=archivebox#r">UnRaid</a></li>
 <li><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a></li>
 <li><a href="https://github.com/YunoHost-Apps/archivebox_ynh">Yunohost</a></li>
@@ -432,7 +431,7 @@ For more discussion on third-party hosting options see here: <a href="https://gi
 #### ➡️&nbsp; Next Steps
 
 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
-- (Optional) [Import browser cookies and settings into a persona](docs/Chromium-Install.md#import-an-existing-browser-profile).
+- (Optional) [Import browser cookies and settings into a persona](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile).
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -614,7 +613,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
 <sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
 <br/><br/>
 <a href="https://demo.archivebox.io">DEMO: <code>https://demo.archivebox.io</code></a><br/>
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#Caveats">Caveats</a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#caveats">Caveats</a>
 <br/>
 </div>
 
@@ -642,7 +641,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
   <i>Save directly from the Share menu or Siri/Shortcuts to your server.</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8" height="22px"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files  
-  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file).</i>
+  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#import-a-list-of-urls-from-a-text-file).</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34" height="22px"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)  
   <i>Instructions: <a href="https://support.google.com/chrome/answer/96816?hl=en">Chrome</a>, <a href="https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer">Firefox</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/assets/511499/24ad068e-0fa6-41f4-a7ff-4c26fc91f71a">Safari</a>, <a href="https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows">IE</a>, <a href="https://help.opera.com/en/latest/features/#bookmarks:~:text=Click%20the%20import/-,export%20button,-on%20the%20bottom">Opera</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
@@ -671,7 +670,7 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
 # echo 'https://example.com' | docker compose run --rm -T archivebox add
 ```
 
-See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
+See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage) page for documentation and examples.
 
 It also includes a built-in scheduled import feature with `archivebox schedule`, handled by the same orchestrator that powers `archivebox server`, so you can pull in URLs from RSS feeds and websites regularly without a separate cron container.
 
@@ -682,8 +681,6 @@ It also includes a built-in scheduled import feature with `archivebox schedule`,
 
 ## Output Formats: What ArchiveBox saves for each URL
 
-Snapshot detail pages group output cards into expandable stacks: HTML, Raster, Article text, Embedded media, OCR, and Metadata. HTML, Raster, Article text, and OCR use the preference order defined in [`archivebox/plugins/output_groups.py`](archivebox/plugins/output_groups.py); media and metadata put the largest output directory first. Unclassified plugins and auxiliary files remain available under Other files. The same layout is included in portable static exports.
-
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ace0954a-ddac-4520-9d18-1c77b1ec50b2" width="330px" align="right" style="float: right"/>
 
 
@@ -814,7 +811,7 @@ Installing directly on <strong>Windows without Docker or WSL/WSL2/Cygwin is not
 
 <h4>Learn More</h4>
 <ul>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Wiki: Install (Dependencies)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">Wiki: Install (Dependencies)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Wiki: Chromium Install</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Wiki: Upgrading or Merging Archives</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Wiki: Troubleshooting (Installing)</a></li>
@@ -868,7 +865,7 @@ Each snapshot subfolder includes static metadata and plain extractor output file
 <h4>Learn More</h4>
 <ul>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Wiki: Setting Up Storage (SMB, NFS, S3, B2, Google Drive, etc.)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
@@ -954,7 +951,7 @@ archivebox manage createsuperuser
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Wiki: Security Overview</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Wiki: Chromium Install (Setting Up a User Profile)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#default_persona">Wiki: Configuration (<code>DEFAULT_PERSONA</code>)</a></li>
 </ul>
 
@@ -1007,7 +1004,7 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active
 
 
 <ul>
-<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
+<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
 <li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
@@ -1068,7 +1065,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <h4>Learn More</h4>
 
 <ul>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user">Wiki: Security Overview (Privilege Dropping)</a></li>
@@ -1193,7 +1190,7 @@ By having users store their content locally or within their organizations, we ca
 
 <h3>Comparison With Other Self-Hosted Archiving Options</h3>
 
-ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
+ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
 
 
 *If you want better fidelity for very complex interactive pages with heavy JS/streams/API requests, check out [ArchiveWeb.page](https://archiveweb.page) and [ReplayWeb.page](https://replayweb.page).*
@@ -1202,7 +1199,7 @@ ArchiveBox differentiates itself from [similar self-hosted projects](https://git
 
 *If you need more advanced recursive spider/crawling ability beyond `--depth=1`, check out [Browsertrix](https://github.com/webrecorder/browsertrix-crawler), [Photon](https://github.com/s0md3v/Photon), or [Scrapy](https://scrapy.org/) and pipe the outputted URLs into ArchiveBox.*
 
-For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects)...
+For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects)...
 
 ArchiveBox is neither the highest fidelity nor the simplest tool available for self-hosted archiving, rather it's a jack-of-all-trades that tries to do most things well by default. We encourage you to try these other tools made by our friends if ArchiveBox isn't suited to your needs.
 
@@ -1257,7 +1254,7 @@ We use the [ArchiveBox GitHub Wiki](https://github.com/ArchiveBox/ArchiveBox/wik
 - [Usage](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage)
 - [Configuration](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)
 - [Supported Sources](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
-- [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox/wiki#can-save-these-things-for-each-site)
+- [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox#output-formats)
 - [Scheduled Archiving](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving)
 
 ## Advanced

archivebox/api/apps.py

@@ -7,6 +7,11 @@ class APIConfig(AppConfig):
     name = "archivebox.api"
     label = "api"
 
+    def ready(self):
+        from archivebox.api.webhooks import register_delete_webhook_field_loader
+
+        register_delete_webhook_field_loader()
+
 
 def register_admin(admin_site):
     from archivebox.api.admin import register_admin

archivebox/api/v1_api.py

@@ -2,6 +2,8 @@ __package__ = "archivebox.api"
 
 
 from io import StringIO
+from functools import lru_cache
+from typing import Any
 from traceback import format_exception
 from contextlib import redirect_stdout, redirect_stderr
 
@@ -10,6 +12,7 @@ from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, Permissio
 from django.contrib.auth.models import User
 
 from ninja import NinjaAPI, Swagger
+from ninja.openapi.schema import OpenAPISchema
 
 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
 
@@ -49,6 +52,16 @@ def register_urls(api: NinjaAPI) -> NinjaAPI:
 
 
 class NinjaAPIWithIOCapture(NinjaAPI):
+    def get_openapi_schema(self, *, path_prefix: str | None = None, path_params: dict[str, Any] | None = None) -> OpenAPISchema:
+        prefix = path_prefix if path_prefix is not None else self.get_root_path(path_params or {})
+        return self._schema_for_prefix(prefix)
+
+    @lru_cache(maxsize=8)
+    def _schema_for_prefix(self, path_prefix: str) -> OpenAPISchema:
+        # Routes are registered at startup. Server discovery can reuse their
+        # schema while preserving distinct URL prefixes for mounted instances.
+        return super().get_openapi_schema(path_prefix=path_prefix)
+
     def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
         stdout, stderr = StringIO(), StringIO()
 

archivebox/api/v1_core.py

@@ -32,7 +32,7 @@ from archivebox.core.models import Snapshot, ArchiveResult, Tag
 from archivebox.core.permissions import public_snapshots_queryset
 from archivebox.api.auth import authenticated_user_from_request
 from archivebox.config.common import get_config
-from archivebox.core.routes_util import build_web_url
+from archivebox.core.routes_util import build_snapshot_detail_url, build_web_url
 from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
 from archivebox.core.tag_util import (
     add_snapshot_counts,
@@ -211,11 +211,22 @@ class ArchiveResultFilterSchema(FilterSchema):
     created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
 
 
+def _with_archiveresult_processes(queryset):
+    # cmd/pwd/version delegate to Process and Binary. Without this join a full
+    # capture's metadata response performs two extra indexed lookups per hook
+    # (roughly 100 queries), despite the initial Snapshot UUID lookup being fast.
+    # Logs and the hydrated hook environment are not response fields; loading
+    # those potentially large values would replace round trips with excess I/O.
+    return queryset.select_related("process__binary").defer("process__stdout", "process__stderr", "process__env")
+
+
 @router.get("/archiveresults", response=list[ArchiveResultSchema], url_name="get_archiveresult")
 @paginate(CustomPagination)
 def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
     """List all ArchiveResult entries matching these filters."""
-    queryset = filters.filter(ArchiveResult.objects.all())
+    queryset = filters.filter(
+        _with_archiveresult_processes(ArchiveResult.objects.all()).prefetch_related("snapshot__crawl__created_by", "snapshot__tags"),
+    )
     if filters.search or filters.snapshot_tag:
         return queryset.distinct()
     return queryset
@@ -223,26 +234,31 @@ def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterS
 
 def _uuid_ref_query(field_name: str, ref: str) -> Q:
     raw_ref = str(ref or "").strip()
-    query = Q(**{f"{field_name}__startswith": raw_ref})
-    if raw_ref:
-        query |= Q(**{f"{field_name}__icontains": raw_ref})
     try:
         parsed_uuid = UUID(raw_ref)
     except (TypeError, ValueError):
+        query = Q(**{f"{field_name}__startswith": raw_ref})
+        if raw_ref:
+            query |= Q(**{f"{field_name}__icontains": raw_ref})
         normalized_ref = raw_ref.replace("-", "")
         if normalized_ref and normalized_ref != raw_ref:
             query |= Q(**{f"{field_name}__startswith": normalized_ref})
             query |= Q(**{f"{field_name}__icontains": normalized_ref})
-    else:
-        query |= Q(**{field_name: parsed_uuid})
-        query |= Q(**{f"{field_name}__startswith": parsed_uuid.hex})
-    return query
+        return query
+    # An exact UUID cannot need a substring fallback. OR-ing it with LIKE
+    # prevents an indexed lookup and made Cabbage metadata reads take seconds.
+    return Q(**{field_name: parsed_uuid})
 
 
 @router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
 def get_archiveresult(request: HttpRequest, archiveresult_id: str):
     """Get a specific ArchiveResult by id."""
-    return ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
+    return (
+        _with_archiveresult_processes(ArchiveResult.objects.all())
+        .select_related("snapshot__crawl__created_by")
+        .prefetch_related("snapshot__tags")
+        .get(_uuid_ref_query("id", archiveresult_id))
+    )
 
 
 def _normalize_uploaded_archiveresult_plugin(plugin: str) -> str:
@@ -350,6 +366,12 @@ def _summarize_archiveresult_output_files(output_files: dict[str, dict[str, Any]
 
 def _get_snapshot_by_ref(snapshot_id: str):
     queryset = Snapshot.objects.select_related("crawl__created_by")
+    try:
+        full_id = UUID(snapshot_id)
+    except (TypeError, ValueError):
+        pass
+    else:
+        return queryset.get(pk=full_id)
     try:
         return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
     except Snapshot.DoesNotExist:
@@ -392,13 +414,13 @@ def _write_archiveresult_files(
     mime_types = _get_archiveresult_upload_form_values(request, "mime_types", "mime_type")
     chunk_output_path = _get_archiveresult_upload_form_value(request, "chunk_output_path")
 
+    output_files = dict(existing_output_files or {})
+    if not files:
+        return output_files
+
     snapshot_dir = snapshot.output_dir
     plugin_dir = snapshot_dir / plugin_name
     storage = FileSystemStorage(location=str(plugin_dir))
-    output_files = dict(existing_output_files or {})
-
-    if not files:
-        return output_files
 
     if chunk_output_path:
         if len(files) != 1:
@@ -630,6 +652,8 @@ def patch_archiveresult(
 class SnapshotSchema(Schema):
     TYPE: str = "core.models.Snapshot"
     id: UUID
+    crawl_id: UUID
+    persona: str | None
     created_by_id: str
     created_by_username: str
     created_at: datetime
@@ -648,6 +672,17 @@ class SnapshotSchema(Schema):
     num_archiveresults: int
     archiveresults: list[MinimalArchiveResultSchema]
 
+    @staticmethod
+    def resolve_archive_path(obj):
+        return obj.archive_path_from_db
+
+    @staticmethod
+    def resolve_persona(obj):
+        # A deduplicated submission can reuse a capture from a different
+        # persona. Report its actual DB-backed owner, not the new request's choice.
+        persona = obj.crawl.resolve_persona()
+        return persona.name if persona else None
+
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by.pk)
@@ -670,12 +705,14 @@ class SnapshotSchema(Schema):
 
     @staticmethod
     def resolve_num_archiveresults(obj, context):
-        return obj.archiveresult_set.all().distinct().count()
+        # This reverse FK contains each result once. DISTINCT would materialize
+        # wide JSON output columns just to count rows through snapshot_id's index.
+        return obj.archiveresult_set.count()
 
     @staticmethod
     def resolve_archiveresults(obj, context):
         if bool(context["request"].__dict__.get("with_archiveresults", False)):
-            return obj.archiveresult_set.all().distinct()
+            return _with_archiveresult_processes(obj.archiveresult_set.all())
         return ArchiveResult.objects.none()
 
 
@@ -700,6 +737,7 @@ class SnapshotDeleteResponseSchema(Schema):
     snapshot_id: str
     crawl_id: str
     deleted_count: int
+    queued_count: int
 
 
 def normalize_tag_list(tags: list[str] | None = None) -> list[str]:
@@ -800,7 +838,7 @@ def _snapshots_rss_response(
     )
 
     for snapshot in snapshots:
-        archived_url = build_web_url(f"/{snapshot.archive_path_from_db}", request=request)
+        archived_url = build_snapshot_detail_url(snapshot.archive_path_from_db, request=request)
         tags = [tag.name for tag in snapshot.tags.all()]
         crawl_user = snapshot.crawl.created_by if snapshot.crawl_id else None
         description = f"Original URL: {snapshot.url}\nArchived snapshot: {archived_url}"
@@ -938,6 +976,15 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     # lock. The unique insert recovery and CAS update below are the request-side
     # coordination boundary for this idempotent metadata sync.
     snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+    if snapshot is None and bool(crawl.get_current_config().get("ONLY_NEW", True)):
+        # Uploading browser metadata is part of the same submission as /add.
+        # It must honor the runner's deduplication policy, even if it arrives
+        # before the runner has processed that crawl. Report the actual owner
+        # so clients never mistake a reused capture for one they just created.
+        snapshot = Snapshot.objects.filter(url=data.url).exclude(status=Snapshot.DELETING_STATE).order_by("-created_at").first()
+        if snapshot is not None:
+            setattr(request, "with_archiveresults", False)
+            return snapshot
     if snapshot is None:
         try:
             snapshot = Snapshot.objects.create(
@@ -971,11 +1018,6 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
             created_by=request.user if isinstance(request.user, User) else None,
         )
 
-    try:
-        snapshot.ensure_crawl_symlink()
-    except Exception:
-        pass
-
     setattr(request, "with_archiveresults", False)
     return snapshot
 
@@ -1037,22 +1079,23 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
 
 @router.delete("/snapshot/{snapshot_id}", response=SnapshotDeleteResponseSchema, url_name="delete_snapshot")
 def delete_snapshot(request: HttpRequest, snapshot_id: str):
+    """Queue deletion; the normal server runner stops writers and removes files.
+
+    The snapshot remains readable with status=deleting until cleanup succeeds.
+    queued_count reports acceptance; deleted_count is zero in this response.
+    """
     snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
     snapshot_id_str = str(snapshot.id)
-    crawl_id_str = str(snapshot.crawl.pk)
-    snapshot.cancel()
-
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        snapshot = get_snapshot(request, snapshot_id_str, with_archiveresults=False)
-        deleted_count, _ = snapshot.delete()
+    crawl_id_str = str(snapshot.crawl_id)
+    # Use the same durable queue as the admin delete action. Stopping writers
+    # and removing archive files belongs to the runner, never the HTTP request.
+    queued_count = Snapshot.objects.filter(pk=snapshot.pk).request_delete()
     return {
         "success": True,
         "snapshot_id": snapshot_id_str,
         "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
+        "deleted_count": 0,
+        "queued_count": queued_count,
     }
 
 

archivebox/api/v1_crawls.py

@@ -182,6 +182,14 @@ def crawl_file(request: HttpRequest, crawl_id: str, path: str):
     if not file_path.is_file() or crawl_root not in file_path.parents:
         raise HttpError(404, "Crawl file not found")
 
+    if path == "chrome_screencast/latest.jpg" and request.GET.get("after"):
+        try:
+            started_at = float(request.GET["after"])
+        except ValueError:
+            raise HttpError(400, "Invalid screencast start time")
+        if file_path.stat().st_mtime < started_at:
+            raise HttpError(404, "No frame for this capture yet")
+
     response = FileResponse(file_path.open("rb"))
     response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
     response["Pragma"] = "no-cache"

archivebox/api/v1_personas.py

@@ -5,9 +5,11 @@ from datetime import datetime
 from typing import Any, Literal
 from uuid import UUID
 
+from django.core.exceptions import ValidationError
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.errors import HttpError
 from ninja.pagination import paginate
 from pydantic import Field
 
@@ -108,10 +110,9 @@ def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrows
 
 
 def find_persona(extension_persona_id: str, name: str) -> Persona | None:
+    named = Persona.find_named(name)
     return (
-        Persona.objects.filter(
-            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
-        )
+        Persona.objects.filter(Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(pk=named.pk if named else None))
         .order_by("created_at")
         .first()
     )
@@ -138,7 +139,11 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     if not is_valid:
         raise ValueError(error_message)
 
-    persona = find_persona(payload.extension_persona_id, name)
+    try:
+        persona = find_persona(payload.extension_persona_id, name)
+        (persona or Persona(name=name)).validate_name(persona.name if persona else name)
+    except ValidationError as err:
+        raise HttpError(409, "; ".join(err.messages)) from err
     created = persona is None
     if persona is None:
         persona = Persona(name=name)

archivebox/api/webhooks.py

@@ -5,12 +5,40 @@ from collections.abc import Callable
 import logging
 
 from django.db import transaction
+from django.db.models.signals import pre_delete
 from signal_webhooks.handlers import sync_task_handler
 
 
 logger = logging.getLogger(__name__)
 
 
+def load_deferred_fields_for_delete_webhook(sender, instance, using, **kwargs) -> None:
+    """Materialize deferred fields while DELETE webhook rows still exist.
+
+    Admin lists defer large fields, but signal_webhooks serializes all concrete
+    fields in post_delete. Loading one then would query an already-deleted row
+    and drop the event. Hydrate only deferred fields on webhook-enabled models
+    here, preserving the normal list-query optimization.
+    """
+    from signal_webhooks.handlers import find_hook_handler
+    from signal_webhooks.utils import reference_for_model
+
+    ref = reference_for_model(type(instance))
+    deferred_fields = instance.get_deferred_fields()
+    if not deferred_fields or find_hook_handler(ref, "DELETE") is None:
+        return
+
+    instance.refresh_from_db(using=using, fields=deferred_fields)
+
+
+def register_delete_webhook_field_loader() -> None:
+    pre_delete.connect(
+        load_deferred_fields_for_delete_webhook,
+        dispatch_uid="archivebox.load_deferred_fields_for_delete_webhook",
+        weak=False,
+    )
+
+
 def warning_error_handler(hook: Any, error: Exception | None) -> None:
     if error is not None:
         logger.warning("Outbound webhook %r failed: %s", hook.name, error)

archivebox/base_models/models.py

@@ -9,6 +9,7 @@ from typing import Any
 from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
 
+from django.core.exceptions import FieldDoesNotExist
 from django.db import models
 from django.db.models import F
 from django.db import transaction
@@ -23,6 +24,110 @@ from django_stubs_ext.db.models import TypedModelMeta
 from archivebox.config import CONSTANTS
 
 
+class PagedQuerySet(models.QuerySet):
+    def paged_iterator(self, chunk_size: int = 500):
+        """
+        Iterate rows using bounded keyset pages instead of one streaming cursor.
+
+        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
+        cursor open until the full queryset is exhausted. That is fine for
+        read-only exports, but update/migration code does filesystem work and
+        writes while iterating; a long-lived read cursor there can stretch lock
+        waits across thousands of rows. This respects the queryset's existing
+        filters, order_by(), select_related(), and prefetch_related() state; if
+        no ordering is defined, it falls back to primary-key order.
+        """
+        pk_field = self.model._meta.pk.name
+        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
+
+        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        ordering = []
+        for term in raw_ordering:
+            descending = term.startswith("-")
+            field_name = term[1:] if descending else term
+            if field_name == "pk":
+                field_name = pk_field
+            ordering.append(f"-{field_name}" if descending else field_name)
+
+        ordered_field_names = [term.removeprefix("-") for term in ordering]
+        try:
+            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
+                offset = 0
+                while True:
+                    batch = list(self[offset : offset + chunk_size])
+                    if not batch:
+                        break
+                    yield from batch
+                    offset += chunk_size
+                return
+        except (AttributeError, FieldDoesNotExist):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
+        if not any(field_name in unique_field_names for field_name in ordered_field_names):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        last_values = None
+        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
+        while True:
+            batch_qs = self.order_by(*ordering)
+            if last_values is not None:
+                page_filter = models.Q()
+                for idx, term in enumerate(ordering):
+                    descending = term.startswith("-")
+                    field_name = term[1:] if descending else term
+                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
+                    comparison = "lt" if descending else "gt"
+                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
+                # Anchor the OR tie-breaks with an explicit leading range so
+                # SQLite/PG can seek the existing order index instead of
+                # rescanning its prefix for every keyset page.
+                first_comparison = "lte" if ordering[0].startswith("-") else "gte"
+                batch_qs = batch_qs.filter(
+                    page_filter,
+                    **{f"{ordered_field_names[0]}__{first_comparison}": last_values[0]},
+                )
+
+            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
+            if not batch_rows:
+                break
+
+            pk_idx = value_field_names.index(pk_field)
+            row_ids = [row[pk_idx] for row in batch_rows]
+            rows_by_id = {obj.pk: obj for obj in self.filter(pk__in=row_ids).order_by()}
+
+            for row in batch_rows:
+                row_id = row[pk_idx]
+                obj = rows_by_id.get(row_id)
+                if obj is not None:
+                    yield obj
+
+            last_values = batch_rows[-1][: len(ordered_field_names)]
+
+
 def normalize_config_json_values(config: Any) -> Any:
     if not isinstance(config, dict):
         return config
@@ -191,9 +296,21 @@ class ModelWithDeleteAfter(models.Model):
         return cls.objects.none()
 
     @classmethod
-    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
+    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True, backfill_cursors: dict | None = None) -> int:
         if backfill_missing:
-            missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
+            # Bound the scanned rows BEFORE evaluating JSON policy/parent
+            # joins. LIMIT on the policy query only bounded its output: finding
+            # no matches could scan the entire history with a SQLite read lock.
+            # The runner keeps a keyset cursor between idle passes, so disabled
+            # policies cannot starve older rows. Restarting safely repeats work.
+            page = cls.objects.order_by("-pk")
+            before = backfill_cursors.get(cls) if backfill_cursors is not None else None
+            if before is not None:
+                page = page.filter(pk__lt=before)
+            page_ids = list(page.values_list("pk", flat=True)[:batch_size])
+            if backfill_cursors is not None:
+                backfill_cursors[cls] = page_ids[-1] if len(page_ids) == batch_size else None
+            missing_delete_at = list(cls.missing_delete_at_candidates().filter(pk__in=page_ids).order_by())
             for obj in missing_delete_at:
                 if obj.set_delete_at_from_config():
                     cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
@@ -231,12 +348,9 @@ class ModelWithOutputDir(ModelWithUUID):
 
     _delete_signal_registered = False
 
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
-        output_dir = Path(self.output_dir)
-        # Avoid holding SQLite write transactions open across slow filesystem work.
-        transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
-        # Note: index.json is deprecated, models should use write_index_jsonl() for full data
+    # Saving metadata must not resolve or create output paths. The runner and
+    # explicit artifact writers create their own directories when writing files;
+    # on_commit would still make HTTP requests wait on remote archive storage.
 
     @property
     def output_dir_parent(self) -> str:
@@ -282,7 +396,14 @@ class ModelWithOutputDir(ModelWithUUID):
             if path.is_symlink() or path.is_file():
                 path.unlink(missing_ok=True)
             elif path.is_dir():
-                shutil.rmtree(path, ignore_errors=True)
+                shutil.rmtree(path)
+            # lstat also catches dangling symlinks and propagates mount and
+            # permission errors rather than mistaking them for absence.
+            try:
+                path.lstat()
+            except FileNotFoundError:
+                continue
+            raise OSError(f"Output path still exists after deletion: {path}")
 
     def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
         """Capture output paths before DB deletion and remove them after commit."""
@@ -294,7 +415,9 @@ class ModelWithOutputDir(ModelWithUUID):
         if cls._delete_signal_registered:
             return
 
-        def schedule_output_dir_cleanup(sender, instance, using, **kwargs):
+        def schedule_output_dir_cleanup(sender, instance, using, origin=None, **kwargs):
+            if getattr(origin, "_output_files_deleted", False):
+                return
             if not isinstance(instance, ModelWithOutputDir):
                 return
             instance.schedule_delete_cleanup(using=using)

archivebox/cli/__init__.py

@@ -2,6 +2,7 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox"
 import os
 import sys
+from contextlib import redirect_stdout
 from importlib import import_module
 
 import rich_click as click
@@ -155,13 +156,10 @@ def cli(ctx, help=False):
     wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
     if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
         try:
-            if subcommand == "server":
-                run_in_debug = "--reload" in sys.argv or os.environ.get("DEBUG") in ("1", "true", "True", "TRUE", "yes")
-                if run_in_debug:
-                    os.environ["ARCHIVEBOX_RUNSERVER"] = "1"
-                    if "--reload" in sys.argv:
-                        os.environ["ARCHIVEBOX_AUTORELOAD"] = "1"
-
+            # RUNSERVER_WORKER sets the web-worker markers on its own child.
+            # Setting them on the server parent leaks them through supervisord
+            # into runners waiting for takeover, which then register themselves
+            # as web workers and trigger spurious runner_watch reloads.
             from archivebox.config.django import setup_django
             from archivebox.misc.checks import check_data_folder, check_migrations
 
@@ -177,8 +175,12 @@ def cli(ctx, help=False):
                 archivebox_init(quick=True)
                 os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
             check_data_folder()
-            if subcommand != "update":
-                check_migrations(auto_apply=True)
+            if subcommand != "update" and not check_migrations(auto_apply=True):
+                from archivebox.machine.models import Machine
+
+                # Host registration and config reconciliation belong to CLI
+                # startup; get_config() is also used by read-only HTTP views.
+                Machine.current()
         except Exception as e:
             STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
             if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
@@ -218,6 +220,12 @@ def main(args=None, prog_name=None):
         raise SystemExit(130) from None
     finally:
         sys.unraisablehook = previous_unraisablehook
+        supervisor_util = sys.modules.get("archivebox.workers.supervisord_util")
+        if supervisor_util is not None:
+            # Short commands can start a foreground-owned search daemon too.
+            # Finish its cleanup before another CLI can borrow a dying stack.
+            with redirect_stdout(sys.stderr):
+                supervisor_util.stop_own_supervisord_process()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_add.py

@@ -6,6 +6,7 @@ __command__ = "archivebox add"
 
 import sys
 import os
+import signal
 from pathlib import Path
 
 from typing import Any, TYPE_CHECKING
@@ -143,7 +144,6 @@ def add(
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
     persona_obj = Persona.get_or_create_named(persona_name)
-    persona_obj.ensure_dirs()
     effective_persona_config = get_config(persona=persona_obj)
 
     crawl_config = {
@@ -216,9 +216,6 @@ def add(
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
-        from archivebox.services.runner import ensure_background_runner
-
-        ensure_background_runner()
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
@@ -227,6 +224,10 @@ def add(
 
         assert command is not None
         exit_code = 0
+        interactive_interrupts = sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())
+
+        def interrupt_hook() -> None:
+            raise KeyboardInterrupt
 
         def crawl_is_complete() -> bool:
             crawl.refresh_from_db(fields=["status"])
@@ -234,7 +235,13 @@ def add(
 
         try:
             try:
-                with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
+                with (
+                    foreground_shutdown_signals(
+                        first_signal_message=None,
+                        interrupt_handlers={signal.SIGINT: interrupt_hook},
+                    ),
+                    foreground_parent_watchdog(),
+                ):
                     while True:
                         standby = standby_until_foreground_runner_needed(
                             command,
@@ -246,7 +253,7 @@ def add(
                         exit_code = run_runner_worker(
                             ["--crawl-id", str(crawl.id)],
                             name=f"worker_runner_add_{os.getpid()}",
-                            interactive_interrupts=True,
+                            interactive_interrupts=interactive_interrupts,
                             config=get_config(crawl=crawl),
                         )
                         crawl.refresh_from_db(fields=["status", "retry_at"])
@@ -311,9 +318,13 @@ def add(
 
             admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
 
-            print("\n[bold]crawl output saved to:[/bold]")
-            print(f"  {rel_output_str}")
-            print(f"  {admin_url}")
+            if snapshots_count:
+                print("\n[bold]snapshot output saved to:[/bold]")
+                for snapshot in crawl.snapshot_set.order_by("created_at").iterator():
+                    click.echo(f"  {snapshot.output_dir}")
+            print("\n[bold]crawl logs and setup:[/bold]")
+            click.echo(f"  {rel_output_str}")
+            click.echo(f"  {admin_url}")
             print(f"\n[bold]total urls snapshotted:[/bold] {snapshots_count}")
             print(f"[bold]total size:[/bold] {total_size}")
             print(f"[bold]total time:[/bold] {duration_str}")
@@ -412,7 +423,23 @@ def main(**kwargs):
             existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
             kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
 
-        add(urls=urls, **kwargs)
+        from archivebox.machine.models import Process
+
+        # CLI commands own their process record and logs. Queue model saves
+        # must not register a process implicitly: HTTP add() shares that path.
+        process = Process.current()
+        exit_code = 0
+        try:
+            add(urls=urls, **kwargs)
+        except BaseException:
+            exit_code = 1
+            raise
+        finally:
+            # --bg/--index-only return without the foreground runner's cleanup.
+            # Leaving them RUNNING makes the active-process index accumulate
+            # historical CLI commands forever, defeating bounded recovery.
+            if process.status == Process.StatusChoices.RUNNING:
+                process.mark_exited(exit_code=exit_code)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_init.py

@@ -111,6 +111,16 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
         else:
             os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
 
+    from archivebox.machine.models import Machine
+
+    # Registration requires the final schema and reconciles legacy file config.
+    Machine.current()
+
+    from archivebox.personas.models import Persona
+
+    # Initialize here, not when /add/ is rendered: a GET must not create profiles.
+    Persona.get_or_create_default().ensure_dirs()
+
     from archivebox.misc.db import database_display_location, is_postgres
 
     assert database_exists()

archivebox/cli/archivebox_install.py

@@ -58,7 +58,6 @@ def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> Non
     from django.utils import timezone
 
     from archivebox.machine.models import Binary, Machine, _canonical_binary_name
-    from archivebox.services.runner import run_due_binary
 
     machine = Machine.current()
     for requested_name in binary_names:
@@ -80,7 +79,13 @@ def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> Non
             binary.status = Binary.StatusChoices.QUEUED
             binary.retry_at = timezone.now()
             binary.save(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
-        run_due_binary(binary, lock_seconds=60)
+        if binary.status == Binary.StatusChoices.INSTALLED:
+            continue
+        # The synchronous CLI needs the lifecycle's verified installation result,
+        # not the scheduler's "claimed work" result. Reuse its failure/requeue and
+        # health-stat handling so a failed install cannot exit successfully.
+        if not binary.install_claimed(lock_seconds=60):
+            raise click.ClickException(f"Binary {binary_name} is already being installed")
 
 
 def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:

archivebox/cli/archivebox_persona.py

@@ -163,8 +163,9 @@ def create_personas(
             rprint(f'[red]Invalid persona name "{name}": {error_msg}[/red]', file=sys.stderr)
             continue
 
-        persona, created = Persona.objects.get_or_create(
-            name=name,
+        created = Persona.find_named(name) is None
+        persona = Persona.get_or_create_named(
+            name,
             defaults={"config": {"PERMISSIONS": permissions}} if permissions else {},
         )
 
@@ -313,7 +314,9 @@ def update_personas(name: str | None = None) -> int:
             if persona_id:
                 persona = Persona.objects.get(id=persona_id)
             else:
-                persona = Persona.objects.get(name=old_name)
+                persona = Persona.find_named(old_name)
+                if persona is None:
+                    raise Persona.DoesNotExist
 
             # Apply updates from CLI flags
             if name:
@@ -325,7 +328,7 @@ def update_personas(name: str | None = None) -> int:
 
                 # Rename the persona directory too
                 old_path = persona.path
-                persona.name = name
+                persona.name = persona.validate_name(name)
                 new_path = persona.path
 
                 if old_path.exists() and old_path != new_path:

archivebox/cli/archivebox_remove.py

@@ -87,9 +87,8 @@ def remove(
     # Delete one snapshot at a time. Each ``.delete()`` is its own short
     # Django-atomic block, so the writer lock is released between rows and
     # an in-flight daemon transaction can interleave instead of deadlocking.
-    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
-    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
-    # rmtree doesn't hold the lock either.
+    # Snapshot.delete() removes and verifies filesystem outputs first, outside
+    # the short database delete transaction. Failed cleanup retains the row.
     #
     deleted_snapshot_pks = []
     timed_out = False

archivebox/cli/archivebox_run.py

@@ -54,12 +54,9 @@ RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
 
 
 def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
-    # A supervised `archivebox run --daemon` is intentionally a disposable
-    # child. If it receives SIGINT/SIGTERM directly, exit with the conventional
-    # signal status so supervisord treats it as an unexpected worker death and
-    # restarts only the runner. The parent `archivebox server` owns supervisord
-    # shutdown and must not be pulled down by a killed daemon worker.
-    os._exit(128 + int(sig))
+    # A supervised daemon must exit non-zero so supervisord restarts it, while
+    # still unwinding the active crawl's hooks and persisted process state.
+    raise SystemExit(128 + sig.value)
 
 
 def process_stdin_records() -> int:
@@ -285,19 +282,26 @@ def run_runner(
         # future ownership lease cannot hide it from the unified scheduler.
         if crawl is not None:
             crawl.update_and_requeue(retry_at=now, refresh=False)
-    # Only a foreground `archivebox add` gets the interactive "abort current
-    # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
-    # orchestrators should shut down immediately and cleanly on the first signal.
-    interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
+    # Interactivity follows terminal ownership, not the spelling of the command.
+    # A supervised add worker has pipes and receives terminal intent/answers from
+    # its parent; direct run owns the terminal itself. Daemons and redirected
+    # commands must never wait for a person who cannot answer their prompt.
+    # Trust the explicit terminal-owner flag, not Process.root: when add borrows
+    # a server-owned supervisord, its worker's OS ancestry still leads to server.
+    # Gating on root=ADD silently made the first Ctrl+C abort after takeover.
+    interactive_interrupts = not daemon and (
+        (sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())) or os.environ.get("ARCHIVEBOX_INTERACTIVE_INTERRUPTS") == "1"
+    )
     if daemon:
         os.environ[RUNNER_DAEMON_ENV] = "1"
 
+    shutdown_state = None
     try:
         with (
             foreground_shutdown_signals(
                 on_signal=_exit_daemon_runner_on_signal if daemon else None,
                 raise_on_first_signal=not daemon,
-            ),
+            ) as shutdown_state,
             foreground_parent_watchdog(enabled=not daemon),
         ):
             run_pending_crawls(
@@ -306,10 +310,16 @@ def run_runner(
                 maintenance_only=maintenance_only,
                 interactive_interrupts=interactive_interrupts,
             )
+        if daemon and shutdown_state.signal_name:
+            return 128 + signal.Signals[shutdown_state.signal_name].value
         return 0
     except KeyboardInterrupt:
-        return 0
+        if daemon and shutdown_state and shutdown_state.signal_name:
+            return 128 + signal.Signals[shutdown_state.signal_name].value
+        return 130
     except asyncio.CancelledError as e:
+        if daemon and shutdown_state and shutdown_state.signal_name:
+            return 128 + signal.Signals[shutdown_state.signal_name].value
         if daemon:
             rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
             return 1
@@ -362,11 +372,11 @@ def main(
             ):
                 sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
         except KeyboardInterrupt:
-            sys.exit(0)
+            sys.exit(130)
 
     with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
         if snapshot_id:
-            sys.exit(run_snapshot_worker(snapshot_id))
+            sys.exit(run_snapshot_worker(snapshot_id, daemon=daemon))
 
         if binary_id:
             try:
@@ -386,7 +396,7 @@ def main(
         if crawl_id:
             sys.exit(
                 run_runner(
-                    daemon=False,
+                    daemon=daemon,
                     crawl_id=crawl_id,
                     maintenance_only=maintenance_only,
                 ),
@@ -401,7 +411,7 @@ def main(
             sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
 
 
-def run_snapshot_worker(snapshot_id: str) -> int:
+def run_snapshot_worker(snapshot_id: str, *, daemon: bool = False) -> int:
     from archivebox.config import CONSTANTS
     from archivebox.core.takeover_util import enter_single_runner_gate
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
@@ -416,6 +426,7 @@ def run_snapshot_worker(snapshot_id: str) -> int:
         return 0
 
     snapshot = None
+    interactive_interrupts = not daemon and sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             for _ in range(10):
@@ -424,7 +435,7 @@ def run_snapshot_worker(snapshot_id: str) -> int:
                     snapshot.update_and_requeue(retry_at=timezone.now())
                 elif snapshot.retry_at > timezone.now():
                     break
-                if not run_due_snapshot(snapshot, lock_seconds=60):
+                if not run_due_snapshot(snapshot, lock_seconds=60, interactive_interrupts=interactive_interrupts):
                     break
         return 0
     except KeyboardInterrupt:
@@ -437,7 +448,7 @@ def run_snapshot_worker(snapshot_id: str) -> int:
                 snapshot.update_and_requeue(retry_at=timezone.now())
         except Exception:
             pass
-        return 0
+        return 130
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
         import traceback

archivebox/cli/archivebox_server.py

@@ -260,6 +260,12 @@ def server(
     bind_spec = next((arg for arg in runserver_args if arg), "")
     host, port = _parse_and_validate_bind_spec(bind_spec)
 
+    from archivebox.personas.models import Persona
+
+    # Older collections deferred this until /add/. Reconcile before serving any
+    # requests, and report ambiguous profile names before Chrome can use them.
+    Persona.get_or_create_default().ensure_dirs()
+
     if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
         from archivebox.core.routes_util import build_admin_url
 

archivebox/cli/archivebox_snapshot.py

@@ -298,7 +298,7 @@ def list_snapshots(
     Exit codes:
         0: Success (even if no results)
     """
-    from archivebox.misc.jsonl import write_record
+    from archivebox.misc.jsonl import write_records
 
     output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
     if output_formats > 1:
@@ -351,9 +351,7 @@ def list_snapshots(
         return 0
 
     if not is_tty:
-        for snapshot_json in iter_snapshot_json(queryset):
-            write_record(snapshot_json)
-            count += 1
+        count = write_records(iter_snapshot_json(queryset))
         rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
         return 0
 

archivebox/cli/archivebox_version.py

@@ -80,20 +80,6 @@ def _render_binary_abspath(abspath: str):
     return Text(abspath, style="green")
 
 
-def _binary_record_matches_runtime(installed, lib_dir: Path) -> bool:
-    if not installed or not installed.is_valid or not installed.version:
-        return False
-    try:
-        abspath = Path(installed.abspath).expanduser().resolve(strict=False)
-        if not abspath.exists():
-            return False
-        if installed.binprovider not in {"", "env", "apt", "brew"}:
-            abspath.relative_to(lib_dir)
-    except (OSError, ValueError):
-        return False
-    return True
-
-
 def _binary_row_dedupe_key(
     *,
     display_name: str,
@@ -115,6 +101,7 @@ def _binary_row_dedupe_key(
 def version(
     quiet: bool = False,
     binaries: Iterable[str] = (),
+    no_cache: bool = False,
 ) -> list[str]:
     """Print the ArchiveBox version, debug metadata, and installed dependency versions"""
 
@@ -127,7 +114,6 @@ def version(
 
     from rich.panel import Panel
     from rich.console import Console
-    from rich.live import Live
     from abx_dl.tables import binary_dependency_status, binary_dependency_table
 
     from archivebox.config import CONSTANTS
@@ -263,7 +249,6 @@ def version(
     prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
     failures = []
     seen_failures: set[str] = set()
-    seen_rows: set[tuple[str, str, str, str]] = set()
 
     from archivebox.plugins.discovery import get_enabled_plugins, get_plugin_catalog
     from abx_dl.config import get_required_binary_requests
@@ -272,37 +257,30 @@ def version(
     from abxpkg.binary_service import BinaryEvent, BinaryService
 
     plugins = get_plugin_catalog()
-    enabled_plugins = plugins.select(get_enabled_plugins(config=config))
-    enabled_plugin_names = set(enabled_plugins)
-    runtime_config = normalize_runtime_config(config.for_crawl(), json_safe=False)
+    enabled_plugin_names = set(get_enabled_plugins(config=config))
+    runtime_config = normalize_runtime_config(dict(config.items()), json_safe=False)
     derived_config: dict[str, object] = {}
-    db_binaries = {}
-    db_available = False
     if in_data_dir:
         try:
             from archivebox.config.django import setup_django
 
             setup_django()
 
-            from archivebox.machine.models import Machine, Binary
+            from archivebox.machine.models import Machine
 
             machine = Machine.current()
             derived_config = normalize_runtime_config(machine.config, json_safe=False)
-            for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
-                if _binary_record_matches_runtime(binary, config.ABXPKG_LIB_DIR):
-                    db_binaries.setdefault(binary.name, binary)
-            db_available = True
 
         except Exception as e:
             prnt()
-            prnt("", f"[yellow]Warning: Could not query collection binary records; resolving through abxpkg: {e}[/yellow]")
+            prnt("", f"[yellow]Warning: Could not query collection machine config; resolving through abxpkg: {e}[/yellow]")
 
     declared_binary_specs: dict[str, dict[str, object]] = {}
+    binary_records_by_plugin: dict[str, list[dict[str, object]]] = {}
     for plugin_name, plugin in plugins.items():
         if not plugin_may_have_requested_binary(plugin):
             continue
-        plugin_requested = bool(requested_names)
-        plugin_enabled = plugin_name in enabled_plugin_names
+        plugin_requested = plugin_name.lower() in {name.lower() for name in requested_names}
         binary_records = get_required_binary_requests(
             plugin,
             plugin.config.required_binaries,
@@ -310,6 +288,10 @@ def version(
             derived_overrides=derived_config,
             run_output_dir=CONSTANTS.DATA_DIR,
         )
+        # Render the exact requests resolved below. Rehydrating all schemas for
+        # display doubled config work, and could describe different requests if
+        # a config file changed while dependency resolution was running.
+        binary_records_by_plugin[plugin_name] = binary_records
         for binary_record in binary_records:
             actual_name = str(binary_record["name"])
             logical_name = (
@@ -320,10 +302,13 @@ def version(
             display_name = logical_name
             if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
                 continue
-            if not plugin_enabled and not requested_names:
-                continue
-            if _binary_record_matches_runtime(db_binaries.get(logical_name), config.ABXPKG_LIB_DIR):
-                continue
+            # Resolve through abxpkg, never the DB's last-known installation row.
+            # Its cache validates current paths, permissions, package fingerprints
+            # and runtime options. Forcing cold resolution here rehashed every
+            # large binary and reran every version subprocess on each diagnostic,
+            # even in an image that had already validated all of them at build.
+            # Keep an explicit full recheck for diagnosing a suspect installation.
+            binary_record["no_cache"] = no_cache
             signature = json.dumps(binary_record, sort_keys=True, default=str)
             declared_binary_specs.setdefault(signature, binary_record)
 
@@ -341,126 +326,41 @@ def version(
 
         loaded_binaries = asyncio.run(resolve_declared_binaries())
 
-    rows: list[dict[str, object]] = []
+    rows_by_binary: dict[str, dict] = {}
     any_rows = False
     any_available = False
     compact_paths = console.is_terminal
-    live_enabled = console.is_terminal
-    live_cm = Live(binary_dependency_table(rows), console=console, refresh_per_second=8) if live_enabled else None
-
-    def emit_row(row: dict[str, object]) -> None:
-        rows.append(row)
-        if live_cm is not None:
-            live_cm.update(binary_dependency_table(rows), refresh=True)
-
-    if live_cm is not None:
-        live_cm.start()
-    try:
-        for plugin_name, plugin in plugins.items():
-            if not plugin_may_have_requested_binary(plugin):
-                continue
-            plugin_requested = bool(requested_names)
-            plugin_enabled = plugin_name in enabled_plugin_names
-            binary_records = get_required_binary_requests(
-                plugin,
-                plugin.config.required_binaries,
-                overrides=runtime_config,
-                derived_overrides=derived_config,
-                run_output_dir=CONSTANTS.DATA_DIR,
+    for plugin_name, plugin in plugins.items():
+        if not plugin_may_have_requested_binary(plugin):
+            continue
+        plugin_requested = plugin_name.lower() in {name.lower() for name in requested_names}
+        plugin_enabled = plugin_name in enabled_plugin_names
+        disabled_by = ", ".join(
+            f"{dependency.enabled_key}=False"
+            for dependency in plugins.select([plugin_name]).values()
+            if not config.get(dependency.enabled_key, True)
+        )
+        binary_records = binary_records_by_plugin[plugin_name]
+        for binary_record in binary_records:
+            binary_record["no_cache"] = no_cache
+            actual_name = str(binary_record["name"])
+            logical_name = (
+                Path(actual_name).expanduser().name
+                if ("/" in actual_name or "\\" in actual_name or actual_name.startswith("~"))
+                else actual_name
             )
-            for binary_record in binary_records:
-                actual_name = str(binary_record["name"])
-                logical_name = (
-                    Path(actual_name).expanduser().name
-                    if ("/" in actual_name or "\\" in actual_name or actual_name.startswith("~"))
-                    else actual_name
-                )
-                display_name = logical_name
-                if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
-                    continue
+            display_name = logical_name
+            if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
+                continue
 
-                installed = db_binaries.get(logical_name) if db_available else None
-                if _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
-                    abspath = installed.abspath
-                    version_str = (installed.version or "unknown")[:15]
-                    provider = (installed.binprovider or "env")[:8]
-                    valid = True
-                elif not plugin_enabled and not requested_names:
-                    # `archivebox version` is expected to verify the active runtime, not
-                    # cold-load every optional plugin provider. Migration and status
-                    # checks often run with PLUGINS narrowed to a tiny set; resolving
-                    # disabled plugin binaries there can spend most of the command on
-                    # providers the current collection will never execute.
-                    continue
-                else:
-                    loaded = loaded_binaries[json.dumps(binary_record, sort_keys=True, default=str)]
-                    abspath = loaded.abspath if loaded is not None else ""
-                    version_str = str(loaded.version or "unknown")[:15] if loaded is not None else "unknown"
-                    provider = str(loaded.binprovider or "env")[:8] if loaded is not None else "env"
-                    valid = loaded is not None
-
-                any_rows = True
-                if valid:
-                    display_path = (
-                        _format_binary_abspath(
-                            abspath,
-                            pwd=Path.cwd(),
-                            lib_dir=config.ABXPKG_LIB_DIR,
-                            personas_dir=CONSTANTS.PERSONAS_DIR,
-                            home=Path.home(),
-                        )
-                        if compact_paths
-                        else abspath
-                    )
-                    rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
-                    any_available = True
-                else:
-                    rendered_path = "[grey53]not installed[/grey53]"
-                    if plugin_enabled and display_name not in seen_failures:
-                        failures.append(display_name)
-                        seen_failures.add(display_name)
-
-                row_key = _binary_row_dedupe_key(
-                    display_name=display_name,
-                    valid=valid,
-                    version=version_str if valid else "-",
-                    provider=provider if valid else "-",
-                    abspath=abspath,
-                )
-                if row_key in seen_rows:
-                    continue
-                seen_rows.add(row_key)
-
-                emit_row(
-                    {
-                        "plugin": plugin_name,
-                        "status": binary_dependency_status(enabled=plugin_enabled, valid=valid),
-                        "binary": display_name,
-                        "version": version_str if valid else "-",
-                        "provider": provider if valid else "-",
-                        "path": rendered_path,
-                        "style": "" if plugin_enabled else "dim",
-                    },
-                )
-
-        if db_available:
-            for binary_name, installed in db_binaries.items():
-                if requested_names and binary_name not in requested_names:
-                    continue
-                abspath = installed.abspath
-                version_str = (installed.version or "unknown")[:15]
-                provider = (installed.binprovider or "env")[:8]
-                row_key = _binary_row_dedupe_key(
-                    display_name=binary_name,
-                    valid=True,
-                    version=version_str,
-                    provider=provider,
-                    abspath=abspath,
-                )
-                if row_key in seen_rows:
-                    continue
-                seen_rows.add(row_key)
+            loaded = loaded_binaries[json.dumps(binary_record, sort_keys=True, default=str)]
+            abspath = loaded.abspath if loaded is not None else ""
+            version_str = str(loaded.version or "unknown") if loaded is not None else "unknown"
+            provider = str(loaded.binprovider or "env") if loaded is not None else "env"
+            valid = loaded is not None and Path(abspath).is_file()
 
+            any_rows = True
+            if valid:
                 display_path = (
                     _format_binary_abspath(
                         abspath,
@@ -472,24 +372,67 @@ def version(
                     if compact_paths
                     else abspath
                 )
-                emit_row(
-                    {
-                        "plugin": "(database)",
-                        "status": binary_dependency_status(enabled=True, valid=True),
-                        "binary": binary_name,
-                        "version": version_str,
-                        "provider": provider,
-                        "path": _render_binary_abspath(display_path) if compact_paths else display_path,
-                        "style": "dim",
-                    },
-                )
-                any_rows = True
+                rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
                 any_available = True
-    finally:
-        if live_cm is not None:
-            live_cm.stop()
-        else:
-            prnt(binary_dependency_table(rows))
+            else:
+                rendered_path = "[grey53]not installed[/grey53]"
+                if plugin_enabled and display_name not in seen_failures:
+                    failures.append(display_name)
+                    seen_failures.add(display_name)
+
+            if not plugin_enabled:
+                rendered_path = f"[grey53]disabled by {disabled_by or 'PLUGINS configuration'}[/grey53]"
+
+            row = {
+                "plugin": plugin_name,
+                "status": binary_dependency_status(enabled=plugin_enabled, valid=valid),
+                "binary": display_name,
+                "version": version_str if valid else "-",
+                "provider": provider if valid else "-",
+                "path": rendered_path,
+                "style": "" if plugin_enabled else "dim",
+                "enabled": plugin_enabled,
+                "valid": valid,
+                "plugins": [plugin_name],
+                "disabled_by": [disabled_by or "PLUGINS configuration"] if not plugin_enabled else [],
+            }
+            existing = rows_by_binary.get(display_name)
+            if existing is None:
+                rows_by_binary[display_name] = row
+            else:
+                consumers = list(dict.fromkeys([*existing["plugins"], plugin_name]))
+                disabled_reasons = list(dict.fromkeys([*existing["disabled_by"], *row["disabled_by"]]))
+                # Disabled consumers cannot mask an active requirement. If any
+                # active request fails (e.g. a version floor), keep that failure.
+                if plugin_enabled and (not existing["enabled"] or not valid):
+                    existing.update(row)
+                existing["plugins"] = consumers
+                existing["disabled_by"] = disabled_reasons
+                existing["plugin"] = ", ".join(consumers)
+                if not existing["enabled"]:
+                    existing["path"] = f"[grey53]disabled by {', '.join(disabled_reasons)}[/grey53]"
+
+    rows = list(rows_by_binary.values())
+
+    if console.is_terminal and console.width < 120:
+        from rich.text import Text
+
+        for row in rows:
+            heading = Text.from_markup(str(row["status"]))
+            heading.append(f" {row['binary']} {row['version']} ({row['provider']}) · {row['plugin']}")
+            prnt(heading, style=str(row["style"]))
+            path = row["path"]
+            prnt(Text.assemble("   ", path if isinstance(path, Text) else Text.from_markup(str(path))), style=str(row["style"]))
+    else:
+        table = binary_dependency_table(rows)
+        table.expand = False
+        table.title = None
+        table.box = None
+        for column in table.columns:
+            column.width = None
+            column.max_width = None
+            column.ratio = None
+        prnt(table)
 
     if not any_rows:
         prnt("", "[grey53]No required binaries declared for discovered plugins.[/grey53]")
@@ -546,6 +489,11 @@ def version(
     is_flag=True,
     help="Only print ArchiveBox version number and nothing else. (equivalent to archivebox --version)",
 )
+@click.option(
+    "--no-cache",
+    is_flag=True,
+    help="Reprobe binary versions and hashes instead of reusing validated provider caches.",
+)
 @click.option(
     "--binaries",
     "-b",

archivebox/cli/rescan.py

@@ -102,9 +102,19 @@ def _reconcile_directory(directory: Path) -> str:
     results = {(item.get("plugin"), item.get("hook_name", "")): item for item in records if item.get("type") == "ArchiveResult"}.values()
     pending = []
     for result in results:
-        result_id = uuid.UUID(result["id"])
         if uuid.UUID(result["snapshot_id"]) != snapshot_id:
             raise ValueError("ArchiveResult belongs to another snapshot")
+        # Legacy JSON conversion produced records without IDs. Recover these
+        # by hook identity while preserving strict collision checks for IDs.
+        if not result.get("id"):
+            if not ArchiveResult.objects.filter(
+                snapshot_id=snapshot_id,
+                plugin=result["plugin"],
+                hook_name=result.get("hook_name", ""),
+            ).exists():
+                pending.append(result)
+            continue
+        result_id = uuid.UUID(result["id"])
         existing = ArchiveResult.objects.filter(pk=result_id).first()
         identity = (snapshot_id, result["plugin"], result.get("hook_name", ""))
         if existing:
@@ -154,7 +164,10 @@ def _reconcile_directory(directory: Path) -> str:
     # Each insert is durable. An interruption after the Snapshot insert is not
     # completion: the next pass still imports missing result IDs and tags.
     for result in pending:
-        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=True)
+        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=bool(result.get("id")))
+    if any(not result.get("id") for result in pending):
+        snapshot.hydrate_archiveresult_output_metadata(snapshot_dir=directory)
+        snapshot.write_index_jsonl(output_dir=directory)
     tags_before = set(snapshot.tags.values_list("name", flat=True))
     snapshot._merge_tags_from_index(record)
     tags_changed = tags_before != set(snapshot.tags.values_list("name", flat=True))
@@ -182,6 +195,10 @@ def reconcile_known_snapshots(snapshots, wait_for_turn=None):
             snapshot.migrate_filesystem_to_current_version()
         directory = Path(snapshot.output_dir)
         if not directory.is_dir():
+            # Canceling queued metadata can seal a row before the runner ever
+            # creates storage. There is nothing on disk to reconcile for it.
+            if snapshot.downloaded_at is None and not snapshot.archiveresult_set.exists():
+                continue
             raise ValueError(f"Missing snapshot directory: {directory}")
         _reconcile_directory(directory)
         for result in snapshot.archiveresult_set.all():

archivebox/config/common.py

@@ -34,6 +34,25 @@ ConfigPayload = dict[str, object]
 PluginSchemaDocuments = dict[str, dict[str, Any]]
 LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 
+
+def _is_cookie_file_config_key(key: str) -> bool:
+    return key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"} or key.endswith("_COOKIES_FILE")
+
+
+def _resolve_cookie_file_paths(config: dict[str, Any]) -> dict[str, Any]:
+    """Keep collection-relative auth paths valid after hooks change cwd."""
+    for key, value in config.items():
+        if _is_cookie_file_config_key(key):
+            if value is None or (isinstance(value, str) and not value.strip()):
+                config[key] = ""
+                continue
+            file_path = Path(value).expanduser()
+            if not file_path.is_absolute():
+                file_path = CONSTANTS.DATA_DIR / file_path
+            config[key] = str(file_path.absolute())
+    return config
+
+
 ###################### Config ##########################
 
 _STDOUT_CONSOLE = Console()
@@ -395,6 +414,36 @@ class ServerConfig(BaseConfigSet):
         )
 
 
+@lru_cache(maxsize=16)
+def _default_sqlite_journal_mode(database_name: str) -> str:
+    if not IN_DOCKER:
+        return "WAL"
+
+    # Native Linux bind mounts share the kernel's locks and mmap; Docker alone
+    # does not make WAL unsafe. DELETE there lets any slow reader block tiny
+    # writes. Desktop VM shares (virtiofs/9p/FUSE), network filesystems and
+    # unknown mounts keep rollback journaling: their host-side readers may not
+    # share SQLite's -shm locking domain. Inspect the DB mount, not archive/.
+    database_path = Path(database_name).expanduser().resolve()
+    try:
+        mounts = Path("/proc/self/mountinfo").read_text().splitlines()
+    except OSError:
+        return "DELETE"
+    filesystem = ""
+    mount_depth = -1
+    for line in mounts:
+        fields, separator, details = line.partition(" - ")
+        if not separator:
+            continue
+        mount_path = fields.split()[4]
+        mount_path = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), mount_path)
+        mount = Path(mount_path)
+        if database_path.is_relative_to(mount) and len(mount.parts) > mount_depth:
+            filesystem = details.split()[0]
+            mount_depth = len(mount.parts)
+    return "WAL" if filesystem in {"ext2", "ext3", "ext4", "xfs", "btrfs", "zfs", "overlay", "tmpfs", "f2fs"} else "DELETE"
+
+
 class DatabaseConfig(BaseConfigSet):
     toml_section_header: str = "DATABASE_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
@@ -410,11 +459,7 @@ class DatabaseConfig(BaseConfigSet):
     DATABASE_USER: str = Field(default="archivebox", alias="ARCHIVEBOX_DATABASE_USER")
     DATABASE_PASSWORD: str = Field(default="", alias="ARCHIVEBOX_DATABASE_PASSWORD")
     SQLITE_JOURNAL_MODE: str = Field(
-        # Docker collections commonly live on a host bind mount. WAL's -shm
-        # locking is only safe when every SQLite process is on the same host;
-        # Docker Desktop/OrbStack place the container and host in different
-        # locking domains and a host-side reader can corrupt the live DB.
-        default="DELETE" if IN_DOCKER else "WAL",
+        default_factory=lambda data: _default_sqlite_journal_mode(str(data["DATABASE_NAME"])),
         alias="ARCHIVEBOX_SQLITE_JOURNAL_MODE",
         pattern=r"(?i)^(DELETE|TRUNCATE|PERSIST|MEMORY|WAL|OFF)$",
     )
@@ -428,11 +473,16 @@ class DatabaseConfig(BaseConfigSet):
     SQLITE_LOCK_RETRY_INTERVAL: float = Field(default=5.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_INTERVAL", gt=0)
 
     @model_validator(mode="after")
-    def reject_docker_sqlite_wal(self):
-        if IN_DOCKER and self.DATABASE_ENGINE.lower() == "sqlite" and self.SQLITE_JOURNAL_MODE.upper() == "WAL":
+    def reject_shared_filesystem_sqlite_wal(self):
+        if (
+            IN_DOCKER
+            and self.DATABASE_ENGINE.lower() == "sqlite"
+            and self.SQLITE_JOURNAL_MODE.upper() == "WAL"
+            and _default_sqlite_journal_mode(self.DATABASE_NAME) != "WAL"
+        ):
             raise ValueError(
-                "SQLITE_JOURNAL_MODE=WAL is unsafe for Docker collections because host bind mounts cross SQLite "
-                "locking domains; use DELETE (the Docker default) or PostgreSQL",
+                "SQLITE_JOURNAL_MODE=WAL requires a local filesystem with shared SQLite locking; "
+                "keep the database on a local Docker volume, use DELETE for host/VM shared mounts, or use PostgreSQL",
             )
         return self
 
@@ -468,7 +518,18 @@ class ArchivingConfig(BaseConfigSet):
     COOKIES_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     AUTH_STORAGE_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
-    URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST")
+    @field_validator("COOKIES_FILE", "AUTH_STORAGE_FILE", mode="before")
+    @classmethod
+    def empty_cookie_file_is_unset(cls, value):
+        return None if isinstance(value, str) and not value.strip() else value
+
+    URL_DENYLIST: str = Field(
+        default=(
+            r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$"
+            r"|^https?://news\.ycombinator\.com/(?:vote|hide)/?(?:[?#]|$)"
+        ),
+        alias="URL_BLACKLIST",
+    )
     URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST")
 
     DEFAULT_PERSONA: str = Field(default="Default", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
@@ -492,7 +553,7 @@ class ArchivingConfig(BaseConfigSet):
             rprint("    (Setting it to somewhere between 30 and 3000 seconds is recommended)", file=sys.stderr)
             rprint(file=sys.stderr)
             rprint("    If you want to make ArchiveBox run faster, disable specific archive methods instead:", file=sys.stderr)
-            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles", file=sys.stderr)
+            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#plugins", file=sys.stderr)
             rprint(file=sys.stderr)
 
     @field_validator("CHECK_SSL_VALIDITY", mode="after")
@@ -819,6 +880,8 @@ class ArchiveBoxBaseConfig(
         if runtime_overrides:
             config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
 
+        _resolve_cookie_file_paths(config)
+
         # Hooks should only see concrete plugin-local flags, never the
         # ArchiveBox selectors used to derive them.
         config.pop("PLUGINS", None)
@@ -843,6 +906,13 @@ class ArchiveBoxBaseConfig(
             lib_dir = CONSTANTS.DATA_DIR / lib_dir
         self.ABXPKG_LIB_DIR = lib_dir.resolve()
 
+        cookie_paths = {key: getattr(self, key) for key in type(self).model_fields if _is_cookie_file_config_key(key)}
+        for key, value in _resolve_cookie_file_paths(cookie_paths).items():
+            if key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"}:
+                setattr(self, key, Path(value) if value else None)
+            else:
+                setattr(self, key, value)
+
         return self
 
     @model_validator(mode="after")
@@ -1107,7 +1177,7 @@ def get_config(
             if apps.ready:
                 from archivebox.machine.models import Machine
 
-                machine = Machine.current()
+                machine = Machine.current_readonly()
         except (DatabaseError, ImportError, RuntimeError, TypeError, ValueError):
             machine = None
 
@@ -1150,11 +1220,15 @@ def get_config(
     scope_overrides: ConfigPayload = {}
 
     if include_machine and machine is not None and machine.config:
-        from archivebox.machine.models import _sanitize_machine_config
+        machine_config = machine.config
+        if resolve_plugins:
+            from archivebox.machine.models import _sanitize_machine_config
+
+            machine_config = _sanitize_machine_config(machine_config, lib_dir=config_data.get("ABXPKG_LIB_DIR"))
 
         scope_overrides.update(
             normalize_runtime_config(
-                _sanitize_machine_config(machine.config, lib_dir=config_data.get("ABXPKG_LIB_DIR")),
+                machine_config,
                 only_crawl_execution=crawl_config_base,
                 exclude_runtime_derived=True,
                 json_safe=False,

archivebox/config/django.py

@@ -63,7 +63,14 @@ def setup_django(check_db=False) -> None:
             os.chown(CONSTANTS.DATA_DIR, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
             if CONSTANTS.DATA_DIR.exists():
                 for child in CONSTANTS.DATA_DIR.iterdir():
-                    os.chown(child, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
+                    try:
+                        os.chown(child, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
+                    except FileNotFoundError:
+                        # A concurrent SQLite transaction can remove its journal
+                        # between directory enumeration and chown. There is no
+                        # ownership left to fix; other permission errors must
+                        # still fail startup rather than being hidden.
+                        continue
 
     # Suppress the "database access during app initialization" warning
     # This warning can be triggered during django.setup() but is safe to ignore

archivebox/config/paths.py

@@ -85,23 +85,31 @@ def get_collection_id(DATA_DIR=DATA_DIR) -> str:
 
 
 @cache
-def get_machine_id() -> str:
-    """Get a short, stable, unique ID for the current machine (e.g. abc45678)"""
+def get_machine_id(length: int = 8) -> str:
+    """Get a machine ID, using the network identity when no OS ID is available."""
+    import machineid
 
-    MACHINE_ID = "unknown"
     try:
-        import machineid
+        return machineid.hashed_id("archivebox")[:length]
+    except machineid.MachineIdNotFound:
+        import uuid
 
-        MACHINE_ID = machineid.hashed_id("archivebox")[:8]
-    except Exception:
-        try:
-            import uuid
-            import hashlib
+        import psutil
 
-            MACHINE_ID = hashlib.sha256(str(uuid.getnode()).encode()).hexdigest()[:8]
-        except Exception:
-            pass
-    return MACHINE_ID
+        node = uuid.getnode()
+        identity = str(node)
+        if node & (1 << 40):
+            # uuid.getnode() silently generates a per-process random value when
+            # tools such as ip/ifconfig are absent. Read interfaces directly so
+            # separate CLI processes agree even in a minimal container.
+            addresses = sorted(
+                address.address
+                for interface in psutil.net_if_addrs().values()
+                for address in interface
+                if address.family == psutil.AF_LINK and address.address not in ("", "00:00:00:00:00:00")
+            )
+            identity = str(int(addresses[0].replace(":", "").replace("-", ""), 16)) if addresses else f"hostname:{socket.gethostname()}"
+        return hashlib.sha256(identity.encode()).hexdigest()[:length]
 
 
 @cache

archivebox/config/views.py

@@ -150,9 +150,9 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     rows = {
         "Binary Name": [],
-        "Found Version": [],
+        "Recorded Version": [],
         "Provided By": [],
-        "Found Abspath": [],
+        "Recorded Abspath": [],
     }
 
     db_binaries = get_db_binaries_by_name()
@@ -160,18 +160,10 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and binary.is_valid)
-
         rows["Binary Name"].append(ItemLink(name, key=name))
-
-        if binary_is_valid:
-            rows["Found Version"].append(f"✅ {binary.version}" if binary.version else "✅ found")
-            rows["Provided By"].append(binary.binprovider or "-")
-            rows["Found Abspath"].append(binary.abspath or "-")
-        else:
-            rows["Found Version"].append("❌ missing")
-            rows["Provided By"].append("-")
-            rows["Found Abspath"].append("-")
+        rows["Recorded Version"].append(binary.version or "-")
+        rows["Provided By"].append(binary.binprovider or "-")
+        rows["Recorded Abspath"].append(binary.abspath or "-")
 
     return TableContext(
         title="Binaries",

archivebox/core/admin_archiveresults.py

@@ -2,11 +2,9 @@ __package__ = "archivebox.core"
 
 import html
 import json
-import os
 import shlex
 from functools import reduce
 from operator import and_
-from pathlib import Path
 from urllib.parse import quote
 
 from django.contrib import admin
@@ -24,7 +22,13 @@ from django.utils.text import smart_split
 
 from archivebox.base_models.admin import BaseModelAdmin
 from archivebox.core.models import ArchiveResult, Snapshot
-from archivebox.core.routes_util import build_snapshot_url
+from archivebox.core.routes_util import (
+    build_snapshot_detail_url,
+    build_snapshot_files_url,
+    build_snapshot_url,
+    build_snapshot_zip_url,
+    get_snapshot_output_anchor,
+)
 from archivebox.core.widgets import InlineTagEditorWidget
 from archivebox.machine.env_util import env_to_shell_exports
 from archivebox.misc.logging_util import printable_filesize
@@ -186,7 +190,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None, can_del
         if embed_path and result.status == "succeeded":
             output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
         else:
-            output_link = build_snapshot_url(snapshot_id, "", config=config)
+            output_link = build_snapshot_detail_url(result.snapshot.archive_path_from_db, config=config)
         output_link_attr = html.escape(output_link, quote=True)
 
         # Get version - try cmd_version field
@@ -555,8 +559,8 @@ class ArchiveResultAdmin(BaseModelAdmin):
             try:
                 queryset = self.get_queryset(request).filter(pk__in=selected)
                 if not queryset.exists():
-                    snapshot = Snapshot.objects.only("id").filter(pk=request.GET.get("snapshot")).first()
-                    return redirect(build_snapshot_url(str(snapshot.id), "index.html", request=request) if snapshot else request.path)
+                    snapshot = Snapshot.objects.filter(pk=request.GET.get("snapshot")).first()
+                    return redirect(build_snapshot_detail_url(snapshot.archive_path_from_db, request=request) if snapshot else request.path)
             except (ValidationError, ValueError):
                 return redirect(request.path)
             return delete_selected(self, request, queryset)
@@ -568,7 +572,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
             request.GET.clear()
             request.META["QUERY_STRING"] = ""
             try:
-                handoff_snapshot = Snapshot.objects.only("id").filter(pk=handoff_snapshot).first()
+                handoff_snapshot = Snapshot.objects.filter(pk=handoff_snapshot).first()
             except (ValidationError, ValueError):
                 handoff_snapshot = None
         saved_list_per_page = self.list_per_page
@@ -582,7 +586,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
                     pk__in=request.POST.getlist(ACTION_CHECKBOX_NAME),
                 ).exists()
             ):
-                return redirect(build_snapshot_url(str(handoff_snapshot.id), "index.html", request=request))
+                return redirect(build_snapshot_detail_url(handoff_snapshot.archive_path_from_db, request=request))
             return response
         finally:
             self.list_per_page = saved_list_per_page
@@ -656,22 +660,40 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     def get_snapshot_view_url(self, result: ArchiveResult) -> str:
         request = self.request
-        return build_snapshot_url(str(result.snapshot_id), request=request, config=request.archivebox_config)
+        return build_snapshot_detail_url(
+            result.snapshot.archive_path_from_db,
+            request=request,
+            config=request.archivebox_config,
+        )
 
     def get_output_view_url(self, result: ArchiveResult) -> str:
         request = self.request
         config = request.archivebox_config
-        output_path = result.embed_path()
-        if not output_path:
-            output_path = result.plugin or ""
-        return build_snapshot_url(str(result.snapshot_id), output_path, request=request, config=config)
+        output_path = get_snapshot_output_anchor(result.plugin, result.embed_path() or "")
+        return build_snapshot_detail_url(
+            result.snapshot.archive_path_from_db,
+            output_path=output_path,
+            request=request,
+            config=config,
+        )
 
     def get_output_files_url(self, result: ArchiveResult) -> str:
         request = self.request
-        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=request.archivebox_config)}/?files=1"
+        return build_snapshot_files_url(
+            str(result.snapshot_id),
+            result.output_folder_path(),
+            request=request,
+            config=request.archivebox_config,
+        )
 
     def get_output_zip_url(self, result: ArchiveResult) -> str:
-        return f"{self.get_output_files_url(result)}&download=zip"
+        request = self.request
+        return build_snapshot_zip_url(
+            str(result.snapshot_id),
+            result.output_folder_path(),
+            request=request,
+            config=request.archivebox_config,
+        )
 
     @admin.display(description="Details", ordering="id")
     def details_link(self, result):
@@ -729,7 +751,11 @@ class ArchiveResultAdmin(BaseModelAdmin):
         request = self.request
         return format_html(
             '<a href="{}"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
+            build_snapshot_detail_url(
+                result.snapshot.archive_path_from_db,
+                request=request,
+                config=request.archivebox_config,
+            ),
             snapshot_id[:8],
             result.snapshot.bookmarked_at.strftime("%Y-%m-%d %H:%M"),
             result.snapshot.url[:128],
@@ -827,11 +853,14 @@ class ArchiveResultAdmin(BaseModelAdmin):
         config = request.archivebox_config
         # Determine output link path - use embed_path() which checks output_files
         embed_path = result.embed_path()
-        output_path = embed_path if (result.status == "succeeded" and embed_path) else "index.html"
-        snapshot_id = str(result.snapshot_id)
+        output_url = (
+            build_snapshot_url(str(result.snapshot_id), embed_path, request=request, config=config)
+            if result.status == "succeeded" and embed_path
+            else build_snapshot_detail_url(result.snapshot.archive_path_from_db, request=request, config=config)
+        )
         return format_html(
             '<a href="{}" class="output-link">↗️</a><pre>{}</pre>',
-            build_snapshot_url(snapshot_id, output_path, request=request, config=config),
+            output_url,
             result.output_str_for_display(),
         )
 
@@ -858,46 +887,16 @@ class ArchiveResultAdmin(BaseModelAdmin):
         )
 
     def output_summary(self, result):
-        snapshot_dir = Path(result.snapshot.output_dir)
-        output_html = format_html(
-            '<pre style="display: inline-block">{}</pre><br/>',
-            result.output_str_for_display(),
-        )
-        snapshot_id = str(result.snapshot_id)
-        request = self.request
-        output_html += format_html(
-            '<a href="{}#all">See result files ...</a><br/><pre><code>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
-        )
-        embed_path = result.embed_path() or ""
-        path_from_embed = snapshot_dir / (embed_path or "")
-        output_html += format_html(
-            '<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>',
-            str(snapshot_dir),
-            str(embed_path),
-        )
-        if os.access(path_from_embed, os.R_OK):
-            root_dir = str(path_from_embed)
-        else:
-            root_dir = str(snapshot_dir)
-
-        for root, dirs, files in os.walk(root_dir):
-            depth = root.replace(root_dir, "").count(os.sep) + 1
-            if depth > 2:
-                continue
-            indent = " " * 4 * (depth)
-            output_html += format_html('<b style="padding: 1px">{}{}/</b><br/>', indent, os.path.basename(root))
-            indentation_str = " " * 4 * (depth + 1)
-            for filename in sorted(files):
-                is_hidden = filename.startswith(".")
-                output_html += format_html(
-                    '<span style="opacity: {}.2">{}{}</span><br/>',
-                    int(not is_hidden),
-                    indentation_str,
-                    filename.strip(),
-                )
-
-        return output_html + mark_safe("</code></pre>")
+        output_html = format_html('<pre style="display: inline-block">{}</pre>', result.output_str_for_display())
+        file_map = result.output_file_map()
+        paths = sorted({path for key in file_map if (path := result.output_file_path(key, output_file_map=file_map))})
+        if paths:
+            output_html += format_html(
+                '<br/><a href="{}">See result files ...</a><pre><code>{}</code></pre>',
+                self.get_output_files_url(result),
+                "\n".join(paths),
+            )
+        return output_html
 
 
 def register_admin(admin_site):

archivebox/core/admin_snapshots.py

@@ -1,8 +1,8 @@
 __package__ = "archivebox.core"
 
 import json
-from functools import lru_cache
-from types import SimpleNamespace
+from math import ceil
+from typing import Any
 
 from django.contrib import admin, messages
 from django.core.exceptions import PermissionDenied
@@ -10,11 +10,12 @@ from django.urls import path, reverse
 from django.shortcuts import get_object_or_404, redirect
 from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed, Http404
 from django.utils import timezone
+from django.utils.formats import date_format
 from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
 from django.db.models import Q, Count, Exists, F, OuterRef, Prefetch
 from django import forms
-from django.template import Template, RequestContext
+from django.template import Context, Template, RequestContext
 from django.contrib.admin.helpers import ActionForm
 
 from archivebox.config.common import get_config
@@ -24,17 +25,32 @@ from archivebox.misc.logging_util import printable_filesize
 from archivebox.misc.serve_static import serve_static_with_byterange_support
 from archivebox.search.admin import SearchResultsAdminMixin, SearchResultsChangeList
 from archivebox.search.views import admin_snapshot_search_stream_view
-from archivebox.core.routes_util import build_snapshot_url, build_web_url
+from archivebox.core.routes_util import (
+    build_snapshot_detail_url,
+    build_snapshot_files_url,
+    build_snapshot_role_output_url,
+    build_snapshot_url,
+    build_snapshot_zip_url,
+    get_snapshot_output_anchor,
+)
 from archivebox.core.tag_util import get_or_create_tag
+from archivebox.core.templatetags.core_tags import snapshot_thumbnail, snapshot_url_text
 from archivebox.plugins.hooks import discover_hooks
-from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_plugins
+from archivebox.plugins.discovery import get_plugin_icon, get_snapshot_role_names
+from archivebox.plugins.output_groups import (
+    OUTPUT_GROUPS,
+    display_plugin_name,
+    output_group_for_plugin,
+    order_output_plugins,
+    plugin_icon_is_hidden,
+    plugin_output_sizes,
+)
 
 from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 
 from archivebox.core.models import Tag, Snapshot, ArchiveResult
 from archivebox.crawls.models import Crawl
 from archivebox.core.admin_archiveresults import render_archiveresults_list
-from archivebox.core.preview_util import PREVIEW_PLUGINS, snapshot_preview_candidates, render_snapshot_preview
 from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
@@ -48,17 +64,14 @@ from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget
 GLOBAL_CONTEXT = {}
 
 SNAPSHOT_PERMISSION_META = PERMISSIONS_META
-GRID_PREVIEW_OUTPUTS = {
-    ("screenshot", "screenshot.png"),
-    ("chrome_extension_screenshot", "screenshot-1.png"),
-    ("chrome_extension_screenshot", "screenshot.png"),
-    ("favicon", "favicon.ico"),
-}
 
 
-@lru_cache(maxsize=1)
-def _plugin_sort_order() -> dict[str, int]:
-    return {get_plugin_name(plugin): idx for idx, plugin in enumerate(get_plugins())}
+def _format_size_column(num_bytes: int) -> str:
+    size = num_bytes
+    for unit in ("B", "KB", "MB", "GB", "TB"):
+        if size < 1024 or unit == "TB":
+            return f"{size:.1f} {unit}" if unit == "GB" else f"{ceil(size)} {unit}"
+        size /= 1024
 
 
 class SnapshotActionForm(ActionForm):
@@ -118,11 +131,11 @@ class SnapshotStatusListFilter(admin.SimpleListFilter):
     parameter_name = "snapshot_status"
 
     def lookups(self, request, model_admin):
-        return Snapshot.StatusChoices.choices
+        return Snapshot._meta.get_field("status").choices
 
     def queryset(self, request, queryset):
         value = self.value()
-        if value in Snapshot.StatusChoices.values:
+        if value in dict(Snapshot._meta.get_field("status").choices):
             return queryset.filter(status=value)
         return queryset
 
@@ -270,27 +283,30 @@ class SnapshotChangeList(SearchResultsChangeList):
         rows = (
             ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
             .order_by("snapshot_id", "plugin")
-            .values_list("snapshot_id", "plugin", "status", "output_size", "output_files")
+            .only("snapshot_id", "plugin", "status", "output_str", "output_size", "output_files")
         )
-        for snapshot_id, plugin, status, output_size, output_files in rows.iterator(chunk_size=1000):
-            if plugin in seen_plugins[snapshot_id]:
+        for result in rows.iterator(chunk_size=1000):
+            if result.plugin in seen_plugins[result.snapshot_id]:
                 continue
-            seen_plugins[snapshot_id].add(plugin)
-            output_results_by_snapshot[snapshot_id].append(
-                SimpleNamespace(plugin=plugin, status=status, output_size=output_size, output_files=output_files),
-            )
+            seen_plugins[result.snapshot_id].add(result.plugin)
+            output_results_by_snapshot[result.snapshot_id].append(result)
 
-        response_results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
-        for result in ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, plugin__in=PREVIEW_PLUGINS).only(
-            "snapshot_id",
-            "plugin",
-            "status",
-            "output_files",
+        card_results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
+        for result in (
+            ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids)
+            .exclude(plugin="")
+            .only(
+                "snapshot_id",
+                "plugin",
+                "status",
+                "output_str",
+                "output_size",
+                "output_files",
+            )
         ):
-            response_results_by_snapshot[result.snapshot_id].append(result)
+            card_results_by_snapshot[result.snapshot_id].append(result)
 
         for obj in self.result_list:
-            obj._preview_results = response_results_by_snapshot[obj.pk]
             counts = status_counts_by_snapshot.get(obj.pk, {})
             total = int(counts.get("total") or 0)
             succeeded = int(counts.get("succeeded") or 0)
@@ -312,6 +328,7 @@ class SnapshotChangeList(SearchResultsChangeList):
                 "is_sealed": obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED, obj.StatusChoices.PAUSED),
             }
             obj.__dict__["_admin_output_results"] = output_results_by_snapshot[obj.pk]
+            obj.__dict__["_snapshot_card_results"] = card_results_by_snapshot[obj.pk]
 
     def get_results(self, request):
         super().get_results(request)
@@ -324,6 +341,7 @@ class SnapshotChangeList(SearchResultsChangeList):
 class SnapshotAdminForm(forms.ModelForm):
     """Custom form for Snapshot admin with tag editor widget."""
 
+    url = forms.CharField(widget=forms.TextInput)
     tags_editor = forms.CharField(
         label="Tags",
         required=False,
@@ -386,7 +404,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     list_display = (
         "permissions_badge",
-        "created_at",
+        "created_at_display",
         "preview_icon",
         "title_str",
         "tags_inline",
@@ -394,7 +412,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         "files",
         "size_with_stats",
     )
-    list_display_links = ("created_at",)
+    list_display_links = ("created_at_display",)
     sort_fields = ("title_str", "created_at", "status", "crawl")
     readonly_fields = (
         "admin_actions",
@@ -508,6 +526,15 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     inlines = []  # Removed TagInline, using TagEditorWidget instead
     list_per_page = 50
 
+    @admin.display(description="Date", ordering="created_at")
+    def created_at_display(self, obj):
+        created_at = timezone.localtime(obj.created_at)
+        return format_html(
+            '<span class="snapshot-created-at"><span class="snapshot-created-at__date">{}</span><span class="snapshot-created-at__time">{}</span></span>',
+            date_format(created_at, "Y-m-d"),
+            date_format(created_at, "g:i A"),
+        )
+
     action_form = SnapshotActionForm
     paginator = AcceleratedPaginator
 
@@ -573,14 +600,15 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     def get_snapshot_view_url(self, obj: Snapshot) -> str:
         request = self.request
-        return build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)
+        return build_snapshot_detail_url(obj.archive_path_from_db, request=request, config=request.archivebox_config)
 
     def get_snapshot_files_url(self, obj: Snapshot) -> str:
         request = self.request
-        return f"{build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)}/?files=1"
+        return build_snapshot_files_url(str(obj.id), request=request, config=request.archivebox_config)
 
     def get_snapshot_zip_url(self, obj: Snapshot) -> str:
-        return f"{self.get_snapshot_files_url(obj)}&download=zip"
+        request = self.request
+        return build_snapshot_zip_url(str(obj.id), request=request, config=request.archivebox_config)
 
     def get_urls(self):
         urls = super().get_urls()
@@ -602,23 +630,17 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return custom_urls + urls
 
     def preview_view(self, request, object_id, plugin, filename):
-        """Serve grid thumbnails from the authenticated admin origin."""
-        if plugin not in PREVIEW_PLUGINS:
-            raise Http404("Unsupported snapshot preview")
+        """Serve a saved image output from the authenticated admin origin."""
         snapshot = get_object_or_404(Snapshot, pk=object_id)
         if not self.has_view_or_change_permission(request, snapshot):
             raise PermissionDenied
-        candidate = next(
-            (
-                candidate
-                for candidate in snapshot_preview_candidates(snapshot)
-                if candidate["plugin"] == plugin and candidate["filename"] == filename
-            ),
-            None,
-        )
-        if candidate is None:
-            raise Http404("Snapshot preview does not exist")
-        output_path = candidate["path"]
+        result = snapshot.archiveresult_set.filter(plugin=plugin, status=ArchiveResult.StatusChoices.SUCCEEDED).first()
+        try:
+            output_path = result.output_file_path(filename) if result else None
+        except (TypeError, ValueError, OverflowError):
+            output_path = None
+        if not output_path:
+            raise Http404("Snapshot image does not exist")
         request.archivebox_cache_policy = "private"
         response = serve_static_with_byterange_support(request, output_path, document_root=snapshot.output_dir)
         if response.status_code != 304 and not response.headers.get("Content-Type", "").lower().startswith("image/"):
@@ -758,7 +780,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
         return qs
 
-    @admin.display(description="👁", ordering="permissions")
+    @admin.display(description="🔒", ordering="permissions")
     def permissions_badge(self, obj):
         permissions = obj.__dict__.get("snapshot_permissions")
         if permissions is None:
@@ -904,7 +926,13 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def status_info(self, obj):
         request = self.request
         config = request.archivebox_config
-        favicon_url = build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)
+        favicon_url = build_snapshot_role_output_url(
+            obj,
+            "list_icon",
+            request=request,
+            config=config,
+            fallback_to_default=True,
+        )
         return format_html(
             """
             Archived: {} ({} files {}) &nbsp; &nbsp;
@@ -942,7 +970,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         show_title = bool(title_raw) and title_normalized != "pending..." and title_normalized != url_normalized
         css_class = "fetched" if show_title else "pending"
 
-        detail_url = build_web_url(f"/{obj.archive_path_from_db}/index.html", request=request, config=config)
+        detail_url = build_snapshot_detail_url(obj.archive_path_from_db, request=request, config=config)
         title_html = ""
         if show_title:
             title_html = format_html(
@@ -952,14 +980,30 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                 urldecode(htmldecode(title_raw))[:128],
             )
 
+        favicon = self._get_favicon_data(obj) or {}
+        favicon_html = (
+            format_html(
+                '<img class="snapshot-url-favicon" src="{}" alt="" loading="lazy" decoding="async" onerror="this.hidden=true">',
+                favicon["url"],
+            )
+            if favicon.get("url")
+            else ""
+        )
         return format_html(
+            '<a class="snapshot-title-detail-hitbox" href="{}" aria-label="Open snapshot details"></a>'
             "{}"
-            '<div style="font-size: 11px; color: #64748b; margin-top: 2px;">'
-            '<a href="{}"><code style="user-select: all;">{}</code></a>'
+            "{}"
+            '<div class="snapshot-title-url">'
+            '<a class="snapshot-original-url" href="{}">{}<code style="user-select: all;">{}</code></a>'
             "</div>",
+            detail_url,
+            format_html('<strong class="snapshot-delete-status">{}</strong>', "Deleting — verifying file removal")
+            if obj.status == Snapshot.DELETING_STATE
+            else "",
             title_html,
             url_raw or obj.url,
-            (url_raw or obj.url)[:128],
+            favicon_html,
+            snapshot_url_text(url_raw or obj.url),
         )
 
     @admin.display(description="Tags", ordering="tag_count")
@@ -986,42 +1030,47 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
         return mark_safe(f'<span class="tags-inline-editor">{tags_html}</span>')
 
-    def _get_preview_data(self, obj):
-        candidates = snapshot_preview_candidates(obj)
-        if not candidates:
-            return None
-        urls = [
-            build_snapshot_url(str(obj.id), candidate["path"], request=self.request, config=self.request.archivebox_config)
-            for candidate in candidates
-        ]
-        favicons = [url for candidate, url in zip(candidates, urls) if candidate["kind"] == "favicon"]
-        return {
-            "img_url": urls[0],
-            "fallback_list": ",".join(urls[1:]),
-            "img_alt": "Favicon" if candidates[0]["kind"] == "favicon" else "Screenshot",
-            "preview_class": "favicon" if candidates[0]["kind"] == "favicon" else "screenshot",
-            "favicon_url": favicons[0] if favicons else "",
-            "favicon_fallback_list": ",".join(favicons[1:]),
-        }
+    def _get_favicon_data(self, obj):
+        list_icon_plugins = set(get_snapshot_role_names("list_icon"))
+        results = self._get_prefetched_results(obj)
+        if results is None:
+            results = obj.archiveresult_set.filter(
+                plugin__in=list_icon_plugins,
+                status=ArchiveResult.StatusChoices.SUCCEEDED,
+            )
+        urls = []
+        for result in results:
+            if result.plugin not in list_icon_plugins or result.status != ArchiveResult.StatusChoices.SUCCEEDED:
+                continue
+            try:
+                output_path = result.embed_path()
+            except (TypeError, ValueError, OverflowError):
+                continue
+            if output_path:
+                urls.append(
+                    build_snapshot_url(str(obj.id), output_path, request=self.request, config=self.request.archivebox_config),
+                )
+        return {"url": urls[0], "fallbacks": ",".join(urls[1:])} if urls else None
 
-    def _render_preview(self, obj, width="100px", height="100px"):
-        return render_snapshot_preview(
+    def _render_thumbnail(self, obj, width="100px", height="100px"):
+        return snapshot_thumbnail(
+            Context({"request": self.request, "CONFIG": self.request.archivebox_config}),
             obj,
-            lambda candidate: reverse("admin:core_snapshot_preview", args=(obj.pk, candidate["plugin"], candidate["filename"])),
+            admin=True,
             width=width,
             height=height,
         )
 
     @admin.display(description="", empty_value="")
     def url_favicon(self, obj):
-        preview = self._get_preview_data(obj)
-        if not preview:
+        favicon = self._get_favicon_data(obj)
+        if not favicon:
             return ""
 
-        favicon_url = preview.get("favicon_url") or ""
+        favicon_url = favicon.get("url") or ""
         if not favicon_url:
             return ""
-        fallback_list = preview.get("favicon_fallback_list") or ""
+        fallback_list = favicon.get("fallbacks") or ""
         onerror_js = (
             "this.dataset.fallbacks && this.dataset.fallbacks.length ? "
             "(this.src=this.dataset.fallbacks.split(',').shift(), "
@@ -1044,8 +1093,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def preview_icon(self, obj):
         return format_html(
             '<a href="{}" title="Open snapshot details">{}</a>',
-            build_web_url(f"/{obj.archive_path_from_db}/index.html", request=self.request, config=self.request.archivebox_config),
-            self._render_preview(obj),
+            build_snapshot_detail_url(obj.archive_path_from_db, request=self.request, config=self.request.archivebox_config),
+            self._render_thumbnail(obj, height="38px"),
         )
 
     @admin.display(description=" ", empty_value="")
@@ -1057,8 +1106,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         size_txt = printable_filesize(archive_size) if archive_size else "pending"
         screenshot_html = format_html(
             '<a href="{}" title="Open snapshot live view" style="display:block;flex:0 0 220px;">{}</a>',
-            build_web_url(f"/{obj.archive_path}", request=request, config=config),
-            self._render_preview(obj, width="220px", height="138px"),
+            build_snapshot_detail_url(obj.archive_path_from_db, request=request, config=config),
+            self._render_thumbnail(obj, width="220px", height="138px"),
         )
 
         return format_html(
@@ -1074,18 +1123,19 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             "</div>",
             screenshot_html,
             size_txt,
-            build_web_url(f"/{obj.archive_path}", request=request, config=config),
-            obj.archive_path,
+            self.get_snapshot_files_url(obj),
+            obj.archive_path_from_db,
         )
 
     @admin.display(
         description="Files Saved",
         ordering="ar_succeeded_count",
+        empty_value="",
     )
     def files(self, obj):
         results = self._get_prefetched_results(obj)
         if results is None:
-            results = obj.archiveresult_set.only("plugin", "status", "output_size")
+            results = obj.archiveresult_set.only("plugin", "status", "output_size", "output_files")
 
         plugins_with_output: dict[str, ArchiveResult] = {}
         for result in results:
@@ -1096,56 +1146,111 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             plugins_with_output.setdefault(result.plugin, result)
 
         if not plugins_with_output:
-            return mark_safe('<span style="opacity: 0.35;">...</span>')
+            return ""
 
-        sorted_results = sorted(
-            plugins_with_output.values(),
-            key=lambda result: (_plugin_sort_order().get(result.plugin, 9999), result.plugin),
-        )
-        visible_results = sorted_results[:14]
-        output = []
+        sorted_results = [plugins_with_output[plugin] for plugin in order_output_plugins(plugins_with_output, plugin_output_sizes(results))]
+        icons_by_group: dict[str, list[dict[str, Any]]] = {group_id: [] for group_id, _, _ in OUTPUT_GROUPS}
         request = self.request
         config = request.archivebox_config
-        for result in visible_results:
+        for result in sorted_results:
+            if plugin_icon_is_hidden(result.plugin):
+                continue
             icon = mark_safe(get_plugin_icon(result.plugin))
             if not icon.strip():
                 continue
-            output.append(
-                format_html(
-                    '<a href="{}" class="exists-True" title="{}">{}</a>',
-                    build_web_url(f"/{obj.archive_path_from_db}/{result.plugin}/", request=request, config=config),
-                    result.plugin,
-                    icon,
-                ),
-            )
-        if len(sorted_results) > len(visible_results):
-            output.append(
-                format_html(
-                    '<span title="{} more outputs">+{}</span>',
-                    len(sorted_results) - len(visible_results),
-                    len(sorted_results) - len(visible_results),
-                ),
+            embed_path = result.embed_path()
+            if not embed_path:
+                continue
+            group_id = output_group_for_plugin(result.plugin)
+            anchor_path = get_snapshot_output_anchor(result.plugin, embed_path)
+            icons_by_group[group_id].append(
+                {
+                    "plugin": result.plugin,
+                    "icon": icon,
+                    "href": build_snapshot_detail_url(
+                        obj.archive_path_from_db,
+                        output_path=anchor_path,
+                        request=request,
+                        config=config,
+                    ),
+                },
             )
 
+        output = []
+        for group_id, label, _ in OUTPUT_GROUPS:
+            icons = icons_by_group[group_id]
+            if icons:
+                top = icons[0]
+                front_index = 0
+                pile_column = len(output)
+                top_html = format_html(
+                    '<a href="{}" class="exists-True files-icon-pile-top files-icon-pile-cover files-icon-plugin--{}" aria-label="{}" data-tooltip="{}" style="--files-icon-front-index:{};--files-icon-front-offset:{}px">{}</a>',
+                    top["href"],
+                    top["plugin"],
+                    label,
+                    label,
+                    front_index,
+                    front_index * 2,
+                    top["icon"],
+                )
+                members = []
+                for icon_index, item in enumerate(icons):
+                    row = icon_index // 5
+                    row_count = min(5, len(icons) - row * 5)
+                    member_top = 8 + row * 18
+                    row_offset = (5 - row_count) * 11 + 3
+                    member_left = 3 + row_offset + (icon_index % 5) * 22
+                    members.append(
+                        format_html(
+                            '<a href="{}" class="exists-True files-icon-plugin--{}" aria-label="{}" data-tooltip="{}" style="--files-icon-index:{};--files-icon-row:{};--files-icon-column:{};--files-icon-row-offset:{}px;--files-icon-member-left:{}px;--files-icon-member-top:{}px">{}</a>',
+                            item["href"],
+                            item["plugin"],
+                            display_plugin_name(item["plugin"]),
+                            item["plugin"],
+                            icon_index,
+                            row,
+                            icon_index % 5,
+                            row_offset,
+                            member_left,
+                            member_top,
+                            item["icon"],
+                        ),
+                    )
+                popup = format_html(
+                    '<span class="files-icon-pile-popup" popover="manual">{}<span class="files-icon-pile-label">{}</span></span>',
+                    mark_safe("".join(str(icon) for icon in members)),
+                    label,
+                )
+                pile_contents = format_html("{}{}", top_html, popup)
+                output.append(
+                    format_html(
+                        '<span class="files-icon-pile files-icon-pile--{}" role="group" aria-label="{} saved outputs" style="--files-icon-popup-left:{}px;--files-icon-rows:{};--files-icon-card-width:{}px">{}</span>',
+                        group_id,
+                        label,
+                        pile_column * -22 - front_index * 2,
+                        (len(icons) + 4) // 5,
+                        116,
+                        pile_contents,
+                    ),
+                )
+
         return format_html(
-            '<span class="files-icons files-icons--compact" style="font-size: 1em; opacity: 0.8;">{}</span>',
+            '<span class="files-icons files-icons--compact" style="font-size: 1em;">{}</span>',
             mark_safe("".join(output)),
         )
 
-    @admin.display()
+    @admin.display(empty_value="")
     def size(self, obj):
-        request = self.request
-        config = request.archivebox_config
         archive_size = self._get_progress_stats(obj)["output_size"] or 0
         if archive_size:
-            size_txt = printable_filesize(archive_size)
+            size_txt = _format_size_column(archive_size)
             if archive_size > 52428800:
                 size_txt = mark_safe(f"<b>{size_txt}</b>")
         else:
-            size_txt = mark_safe('<span style="opacity: 0.3">...</span>')
+            return ""
         return format_html(
             '<a href="{}" title="View all files">{}</a>',
-            build_web_url(f"/{obj.archive_path}", request=request, config=config),
+            self.get_snapshot_files_url(obj),
             size_txt,
         )
 
@@ -1215,6 +1320,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     @admin.display(
         description="Size",
         ordering="output_size",
+        empty_value="",
     )
     def size_with_stats(self, obj):
         """Show archive size with output size from archive results."""
@@ -1223,24 +1329,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         size_bytes = output_size or 0
 
         if size_bytes:
-            size_txt = printable_filesize(size_bytes)
+            size_txt = _format_size_column(size_bytes)
             if size_bytes > 52428800:  # 50MB
                 size_txt = mark_safe(f"<b>{size_txt}</b>")
         else:
-            size_txt = mark_safe('<span style="opacity: 0.3">...</span>')
-
-        # Show hook statistics
-        if stats["total"] > 0:
-            return format_html(
-                '<a href="{}" title="View all files" style="white-space: nowrap;">'
-                "{}</a>"
-                '<div style="font-size: 10px; color: #94a3b8; margin-top: 2px;">'
-                "{}/{} hooks</div>",
-                self.get_snapshot_files_url(obj),
-                size_txt,
-                stats["succeeded"],
-                stats["total"],
-            )
+            return ""
 
         return format_html(
             '<a href="{}" title="View all files">{}</a>',
@@ -1368,9 +1461,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     )
     def url_str(self, obj):
         return format_html(
-            '<a href="{}"><code style="user-select: all;">{}</code></a>',
+            '<a class="snapshot-original-url" href="{}"><code style="user-select: all;">{}</code></a>',
             obj.url,
-            obj.url[:128],
+            snapshot_url_text(obj.url),
         )
 
     @admin.display(description="Health", ordering="health")
@@ -1446,25 +1539,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.action(
         description="🗑️ Delete",
+        permissions=["delete"],
     )
     def delete_snapshots(self, request, queryset):
-        """Delete snapshots in a single transaction to avoid SQLite concurrency issues."""
-        from django.db import transaction
-
-        total = queryset.count()
-
-        # Get list of IDs to delete first (outside transaction)
-        ids_to_delete = list(queryset.values_list("pk", flat=True))
-
-        # Delete everything in a single atomic transaction
-        with transaction.atomic():
-            deleted_count, _ = Snapshot.objects.filter(pk__in=ids_to_delete).delete()
-
+        """Keep the rows until the runner removes and verifies their files."""
+        total = queryset.request_delete()
         messages.success(
             request,
-            mark_safe(
-                f"Successfully deleted {total} Snapshots ({deleted_count} total objects including related records). Don't forget to scrub URLs from import logs (data/sources) and error logs (data/logs) if needed.",
-            ),
+            f"Queued {total} snapshots for deletion. They remain visible as Deleting until ArchiveBox verifies their files are removed.",
         )
 
     @admin.action(

archivebox/core/admin_tags.py

@@ -7,12 +7,13 @@ from django.contrib import admin, messages
 from django.contrib.admin.options import IS_POPUP_VAR
 from django.http import HttpRequest, HttpResponseRedirect
 from django.template.response import TemplateResponse
+from django.db.models import Prefetch
 from django.urls import reverse
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 
 from archivebox.base_models.admin import BaseModelAdmin
-from archivebox.core.models import SnapshotTag, Tag
+from archivebox.core.models import ArchiveResult, SnapshotTag, Tag
 from archivebox.core.tag_util import (
     TAG_HAS_SNAPSHOTS_CHOICES,
     TAG_SORT_CHOICES,
@@ -24,7 +25,8 @@ from archivebox.core.tag_util import (
     normalize_has_snapshots_filter,
     normalize_tag_sort,
 )
-from archivebox.core.routes_util import build_snapshot_url
+from archivebox.core.routes_util import build_snapshot_role_output_url
+from archivebox.plugins.discovery import get_snapshot_role_names
 
 
 class TagInline(admin.TabularInline):
@@ -191,7 +193,26 @@ class TagAdmin(BaseModelAdmin):
 
     @admin.display(description="Snapshots")
     def snapshots(self, tag: Tag):
-        snapshots = tag.snapshot_set.select_related("crawl__created_by").order_by("-downloaded_at", "-created_at", "-pk")[:10]
+        snapshots = (
+            tag.snapshot_set.select_related("crawl__created_by")
+            .prefetch_related(
+                Prefetch(
+                    "archiveresult_set",
+                    queryset=ArchiveResult.objects.filter(
+                        plugin__in=get_snapshot_role_names("list_icon"),
+                        status=ArchiveResult.StatusChoices.SUCCEEDED,
+                    ).only(
+                        "snapshot_id",
+                        "plugin",
+                        "status",
+                        "output_str",
+                        "output_files",
+                    ),
+                    to_attr="_list_icon_results",
+                ),
+            )
+            .order_by("-downloaded_at", "-created_at", "-pk")[:10]
+        )
         total_count = tag.snapshot_set.count()
         if not snapshots:
             return mark_safe(
@@ -214,7 +235,12 @@ class TagAdmin(BaseModelAdmin):
                 </a>
                 """,
                     reverse("admin:core_snapshot_change", args=[snapshot.pk]),
-                    build_snapshot_url(str(snapshot.pk), "favicon.ico"),
+                    build_snapshot_role_output_url(
+                        snapshot,
+                        "list_icon",
+                        archive_results=snapshot._list_icon_results,
+                        fallback_to_default=True,
+                    ),
                     title[:120],
                     snapshot.url[:120],
                 ),

archivebox/core/asgi.py

@@ -31,7 +31,7 @@ django_application = ArchiveBoxASGIHandler()
 
 async def application(scope, receive, send):
     if scope["type"] == "websocket":
-        from archivebox.opencode.views import websocket_view
+        from abx_plugins.plugins.opencode.archivebox.views import websocket_view
 
         return await websocket_view(scope, receive, send)
     return await django_application(scope, receive, send)

archivebox/core/forms.py

@@ -304,13 +304,13 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             self.can_override_crawl_config = bool(user.is_authenticated and user.is_active and user.is_superuser)
         super().__init__(*args, **kwargs)
 
-        default_persona = Persona.get_or_create_default()
+        # Persona creation belongs to init/server startup, never a GET of /add/.
         persona_queryset = Persona.objects.order_by("name")
         if not self.can_override_crawl_config:
             persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
         self.fields["persona"].queryset = persona_queryset
 
-        selected_persona = persona_queryset.filter(id=default_persona.id).first() or persona_queryset.first()
+        selected_persona = persona_queryset.filter(name__iexact="Default").first() or persona_queryset.first()
         default_config = get_config(persona=selected_persona) if selected_persona else get_config()
         if selected_persona:
             self.fields["persona"].initial = selected_persona.name
@@ -338,6 +338,12 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
                 get_choice_field(self, field_name).choices = [(plugin.name, plugin.name) for plugin in grouped_plugins.get(category, [])]
             self.plugin_groups = []
 
+    def clean_persona(self):
+        persona = self.cleaned_data.get("persona")
+        if persona:
+            persona.validate_name(persona.name)
+        return persona
+
     def clean(self):
         cleaned_data = super().clean() or {}
 

archivebox/core/middleware.py

@@ -33,6 +33,7 @@ from archivebox.core.routes_util import (
     split_host_port,
 )
 from archivebox.core.views import OriginalDomainHostView, SnapshotHostView
+from archivebox.misc.serve_static import FAVICON_CACHE_CONTROL
 
 ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in"
 
@@ -154,8 +155,21 @@ def CacheControlMiddleware(get_response):
                     response.headers["Last-Modified"] = http_date(mtime)
                 return response
 
-        # Keep artifact freshness/validators, but never let a shared cache store
-        # private or unlisted snapshots. Auth redirects and errors stay unstored.
+        if response.status_code in (200, 206, 304) and response.get("Cache-Control") == FAVICON_CACHE_CONTROL:
+            # The same raw favicon bytes are shared across authenticated viewers.
+            # Session middleware may have added Cookie to Vary during authorization.
+            vary = [
+                value.strip()
+                for value in response.get("Vary", "").split(",")
+                if value.strip().lower() not in {"", "cookie", "authorization"}
+            ]
+            if vary:
+                response["Vary"] = ", ".join(vary)
+            else:
+                response.headers.pop("Vary", None)
+            return response
+
+        # Other private/unlisted artifacts remain private; auth errors stay unstored.
         snapshot_policy = getattr(request, "archivebox_cache_policy", None)
         if snapshot_policy and response.status_code in (200, 206, 304):
             if not response.get("Cache-Control"):

archivebox/core/migrations/0024_assign_default_crawl.py

@@ -100,11 +100,12 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor):
     if schema_editor.connection.vendor != "sqlite":
         return
 
-    from django.db import connection
+    from django.utils import timezone
     import uuid as uuid_lib
-    from datetime import datetime
 
+    connection = schema_editor.connection
     cursor = connection.cursor()
+    now = connection.ops.adapt_datetimefield_value(timezone.now())
 
     # Check if there are any snapshots without a crawl
     cursor.execute("SELECT COUNT(*) FROM core_snapshot WHERE crawl_id IS NULL")
@@ -125,14 +126,13 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor):
             INSERT INTO auth_user (id, password, is_superuser, username, first_name, last_name, email, is_staff, is_active, date_joined)
             VALUES (1, '!', 1, 'system', '', '', '', 1, 1, %s)
         """,
-            [datetime.now().isoformat()],
+            [now],
         )
 
     # Create a default crawl for migrated snapshots.
     # Depending on migration graph order, later crawls migrations may already
     # have removed output_dir by the time this data migration runs.
     crawl_id = uuid_lib.uuid4().hex
-    now = datetime.now().isoformat()
     cursor.execute("PRAGMA table_info(crawls_crawl)")
     crawl_columns = {row[1] for row in cursor.fetchall()}
     default_columns: list[str] = []
@@ -155,9 +155,9 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor):
             status, retry_at, created_by_id, schedule_id, config, persona_id
         ) VALUES (%s, %s, %s, 0, 0, %s, 0, '', 'Migrated from v0.7.2/v0.8.6',
                   'Auto-created crawl for migrated snapshots', {default_values_sql}
-                  'sealed', %s, 1, NULL, '{{}}', NULL)
+                  'sealed', NULL, 1, NULL, '{{}}', NULL)
     """,
-        [crawl_id, now, now, crawl_urls, now],
+        [crawl_id, now, now, crawl_urls],
     )
 
     # Assign all snapshots without a crawl to the default crawl

archivebox/core/migrations/0055_repair_legacy_retry_timestamps.py

@@ -0,0 +1,60 @@
+"""Repair SQLite queue timestamps written as raw ISO strings by old migrations."""
+
+from django.db import migrations
+from django.utils.dateparse import parse_datetime
+
+
+def repair_legacy_retry_timestamps(apps, schema_editor):
+    connection = schema_editor.connection
+    if connection.vendor != "sqlite":
+        return
+
+    with connection.cursor() as cursor:
+        # 0024 created this Crawl already sealed, but left a due retry marker.
+        # Its orphaned snapshots have their own scheduler rows for maintenance.
+        cursor.execute(
+            """
+            UPDATE crawls_crawl SET retry_at = NULL
+            WHERE status = 'sealed'
+              AND label = 'Migrated from v0.7.2/v0.8.6'
+              AND notes = 'Auto-created crawl for migrated snapshots'
+              AND substr(retry_at, 11, 1) = 'T'
+            """,
+        )
+
+        # SQLite compares stored date strings lexically. A raw `T` timestamp
+        # from a legacy migration sorts differently from Django's space-separated
+        # parameter, so a due row can be selected but never claimed. Preserve
+        # the datetime Django already reads from each row while rewriting its
+        # storage representation through the active database adapter.
+        for table in ("crawls_crawl", "core_snapshot", "machine_binary", "machine_process"):
+            last_id = ""
+            while True:
+                cursor.execute(
+                    f"SELECT id, CAST(retry_at AS TEXT) FROM {table} WHERE id > %s AND substr(retry_at, 11, 1) = 'T' ORDER BY id LIMIT 500",
+                    [last_id],
+                )
+                rows = cursor.fetchall()
+                if not rows:
+                    break
+                for row_id, raw_retry_at in rows:
+                    try:
+                        parsed = parse_datetime(raw_retry_at)
+                    except ValueError:
+                        continue
+                    if parsed is None:
+                        continue
+                    normalized = connection.ops.adapt_datetimefield_value(parsed)
+                    cursor.execute(
+                        f"UPDATE {table} SET retry_at = %s WHERE id = %s AND retry_at = %s",
+                        [normalized, row_id, raw_retry_at],
+                    )
+                last_id = rows[-1][0]
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0054_snapshot_fs_version_index"),
+    ]
+
+    operations = [migrations.RunPython(repair_legacy_retry_timestamps, migrations.RunPython.noop)]

archivebox/core/migrations/0057_repair_migrated_process_links.py

@@ -0,0 +1,25 @@
+"""Repair process links after both core's legacy copy and machine migrations."""
+
+from importlib import import_module
+
+from django.db import migrations
+
+
+repair_process_binary_iface_links = import_module(
+    "archivebox.machine.migrations.0020_repair_process_binary_iface_links",
+).repair_process_binary_iface_links
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("core", "0055_repair_legacy_retry_timestamps"),
+        ("machine", "0022_networkinterface_identity_without_mac"),
+    ]
+
+    operations = [
+        # The original repair can run before core.0027 creates legacy Process
+        # rows. Repair again after the copy, including already upgraded DBs.
+        migrations.RunPython(repair_process_binary_iface_links, migrations.RunPython.noop),
+    ]

archivebox/core/migrations/0058_snapshot_deleting_status.py

@@ -0,0 +1,28 @@
+# Generated by Django 6.1 on 2026-09-30 03:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0057_repair_migrated_process_links"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("queued", "Queued"),
+                    ("started", "Started"),
+                    ("paused", "Paused"),
+                    ("sealed", "Sealed"),
+                    ("deleting", "Deleting"),
+                ],
+                db_index=True,
+                default="queued",
+                max_length=15,
+            ),
+        ),
+    ]

archivebox/core/preview_util.py

@@ -1,114 +0,0 @@
-__package__ = "archivebox.core"
-
-import hashlib
-import json
-from pathlib import PurePosixPath
-from urllib.parse import urlsplit
-
-from django.template.loader import render_to_string
-from django.utils.safestring import mark_safe
-
-EXTENSION_SCREENSHOT_PLUGIN = "chrome_extension_screenshot"
-PREVIEW_PLUGINS = ("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "seo", "responses", "favicon")
-IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".gif", ".avif", ".svg", ".ico"}
-
-
-def positive_int(value):
-    try:
-        return max(0, int(value))
-    except (ValueError, TypeError, OverflowError):
-        return 0
-
-
-def preview_candidates(results):
-    """Select only saved outputs. Never read files or fetch remote metadata."""
-    candidates = []
-    for result in results:
-        files = result.output_files
-        if not isinstance(files, dict):
-            continue
-        for path, info in files.items():
-            if not isinstance(path, str) or not isinstance(info, dict) or not positive_int(info.get("size")):
-                continue
-            if path.startswith("/") or ".." in PurePosixPath(path).parts:
-                continue
-            plugin = result.plugin
-            relative = path.removeprefix(f"{plugin}/")
-            output_path = path if info.get("root_relative") else f"{plugin}/{path}"
-            kind = "image"
-            rank = None
-            if plugin == "screenshot" and relative == "screenshot.png":
-                rank = 0
-            elif plugin == EXTENSION_SCREENSHOT_PLUGIN and relative in ("screenshot-1.png", "screenshot.png"):
-                rank = 1 if relative == "screenshot-1.png" else 2
-            elif (
-                plugin == "seo"
-                and PurePosixPath(relative).stem == "featured-image"
-                and PurePosixPath(relative).suffix.lower() in IMAGE_EXTENSIONS
-            ):
-                rank = 3
-            elif plugin == "responses":
-                mime = str(info.get("mimetype") or info.get("mime_type") or "")
-                if not (relative.startswith("image/") or (relative.startswith("all/") and mime.startswith("image/"))):
-                    continue
-                width, height = positive_int(info.get("width")), positive_int(info.get("height"))
-                if width and height and (min(width, height) <= 2 or max(width, height) <= 64):
-                    continue
-                if PurePosixPath(relative).suffix.lower() == ".ico":
-                    continue
-                rank = 4
-                kind = "response"
-            elif plugin == "favicon" and relative == "favicon.ico":
-                rank = 5
-                kind = "favicon"
-            if rank is None:
-                continue
-            width, height = positive_int(info.get("width")), positive_int(info.get("height"))
-            # Prefer usable known dimensions; byte size remains the legacy fallback.
-            usable = bool(width and height and 0.25 <= width / height <= 4)
-            score = (rank, -int(usable), -(width * height if usable else 0), -positive_int(info.get("size")), output_path)
-            candidates.append((score, {"path": output_path, "kind": kind, "plugin": plugin, "filename": path}))
-    seen = set()
-    selected = []
-    for _score, candidate in sorted(candidates, key=lambda item: item[0]):
-        if candidate["path"] not in seen:
-            selected.append(candidate)
-            seen.add(candidate["path"])
-    return selected
-
-
-def snapshot_preview_candidates(snapshot):
-    if "_preview_candidates" not in snapshot.__dict__:
-        results = snapshot.__dict__.get("_preview_results")
-        if results is None:
-            results = snapshot.__dict__.get("_prefetched_objects_cache", {}).get("archiveresult_set")
-        if results is None:
-            results = snapshot.archiveresult_set.filter(plugin__in=PREVIEW_PLUGINS).only("plugin", "output_files")
-        snapshot._preview_candidates = preview_candidates(results)
-    return snapshot._preview_candidates
-
-
-def render_snapshot_preview(snapshot, url_for_candidate, width="100px", height="100px"):
-    candidates = []
-    for candidate in snapshot_preview_candidates(snapshot):
-        candidates.append({"url": url_for_candidate(candidate), "kind": candidate["kind"]})
-    try:
-        host = urlsplit(snapshot.url).hostname or "Archive"
-    except ValueError:
-        host = "Archive"
-    host = host.removeprefix("www.")
-    hue = int(hashlib.sha256(host.encode()).hexdigest()[:6], 16) % 360
-    return mark_safe(
-        render_to_string(
-            "core/snapshot_thumbnail.html",
-            {
-                "candidates": json.dumps(candidates),
-                "first": candidates[0] if candidates else None,
-                "host": host,
-                "initial": host[:1].upper(),
-                "hue": hue,
-                "width": width,
-                "height": height,
-            },
-        ),
-    )

archivebox/core/recovery_util.py

@@ -74,8 +74,9 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
         **snapshot_filter,
     ).update(retry_at=now, modified_at=now)
 
+    # Find rare uploads through hook_name's index before checking their parents;
+    # a correlated upload probe otherwise visits every sealed snapshot.
     extension_upload_results = ArchiveResult.objects.filter(
-        snapshot_id=OuterRef("pk"),
         hook_name=Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME,
     )
     server_results = ArchiveResult.objects.filter(snapshot_id=OuterRef("pk")).exclude(
@@ -83,15 +84,15 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
     )
     extension_only_snapshots = (
         Snapshot.objects.filter(
+            id__in=extension_upload_results.values("snapshot_id"),
             status=Snapshot.StatusChoices.SEALED,
             crawl__status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED],
             **snapshot_filter,
         )
         .annotate(
-            has_extension_upload=Exists(extension_upload_results),
             has_server_result=Exists(server_results),
         )
-        .filter(has_extension_upload=True, has_server_result=False)
+        .filter(has_server_result=False)
     )
     # Older browser-extension uploads could win a race with runner startup:
     # their successful external rows made the fresh Snapshot look finished

archivebox/core/routes_util.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import re
 from typing import Any
-from urllib.parse import urlparse
+from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlsplit, urlunsplit
 
 from django.utils.http import url_has_allowed_host_and_scheme
 
@@ -11,6 +11,8 @@ from archivebox.config.common import get_config, get_request_config
 _SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
 _SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P<suffix>[0-9a-fA-F]{12})$")
 _ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api")
+_URL_PATH_SAFE = "/@-._~!$&'()*+,;="
+_URL_FRAGMENT_SAFE = "/@-._~!$&'()*+,;="
 
 
 def split_host_port(host: str) -> tuple[str, str | None]:
@@ -60,13 +62,6 @@ def _csrf_trusted_origins(config) -> list[str]:
     return seen
 
 
-def _allowed_hosts(config) -> set[str]:
-    raw = (config.ALLOWED_HOSTS or "").strip()
-    if not raw:
-        return set()
-    return {entry.strip().lower() for entry in raw.split(",") if entry.strip() and entry.strip() != "*"}
-
-
 def derive_base_url_from_csrf(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     """Pick a single CSRF_TRUSTED_ORIGINS entry to act as the implicit BASE_URL.
 
@@ -93,11 +88,6 @@ def get_listen_host(config: dict[str, Any] | None = None, **config_kwargs: Any)
     return (config.BIND_ADDR or "").strip()
 
 
-def get_listen_parts(config: dict[str, Any] | None = None, **config_kwargs: Any) -> tuple[str, str | None]:
-    config = config or get_config(**config_kwargs)
-    return split_host_port(get_listen_host(config=config))
-
-
 def _with_port(host: str, port: str | None) -> str:
     return f"{host}:{port}" if port else host
 
@@ -146,7 +136,7 @@ def canonical_base_host_for_request(request_host: str) -> str:
 
 def _root_host_from_listen(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
-    listen_host, listen_port = get_listen_parts(config=config)
+    listen_host, listen_port = split_host_port(get_listen_host(config=config))
     root_host = "archivebox.localhost" if _is_local_bind_host(listen_host) else listen_host
     return _with_port(root_host, listen_port) if root_host else ""
 
@@ -380,11 +370,6 @@ def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any]
     )
 
 
-def get_original_base_url(domain: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
-    return _build_url(get_web_base_url(request=request, config=config), f"/original/{domain}")
-
-
 def build_admin_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     return _build_url(get_admin_base_url(request, config=config, **config_kwargs), path)
 
@@ -394,11 +379,109 @@ def build_web_url(path: str = "", request=None, config: dict[str, Any] | None =
 
 
 def build_snapshot_url(snapshot_id: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_snapshot_base_url(snapshot_id, request=request, config=config, **config_kwargs), path)
+    # `path` is a filesystem-relative output path, not an already-encoded URL.
+    # In particular, wget can save literal `%3A` in filenames; emitting that
+    # unchanged makes the browser request `:`, which cannot match the file.
+    output_path = quote(str(path).lstrip("/"), safe=_URL_PATH_SAFE)
+    return _build_url(get_snapshot_base_url(snapshot_id, request=request, config=config, **config_kwargs), output_path)
+
+
+def build_snapshot_detail_path(archive_path: str, output_path: str = "", prefix: str = "/") -> str:
+    """Build the live Django snapshot-detail path and optional output anchor."""
+    normalized_archive_path = quote(str(archive_path or "").strip("/"), safe=_URL_PATH_SAFE)
+    detail_path = f"{prefix}{normalized_archive_path}"
+    normalized_output_path = str(output_path or "").lstrip("#/")
+    if not normalized_output_path:
+        return detail_path
+    fragment = quote(normalized_output_path, safe=_URL_FRAGMENT_SAFE)
+    return f"{detail_path}#{fragment}"
+
+
+def build_snapshot_index_path(archive_path: str, output_path: str = "", prefix: str = "/") -> str:
+    """Build a path to the literal exported ``index.html`` snapshot file."""
+    normalized_archive_path = quote(str(archive_path or "").strip("/"), safe=_URL_PATH_SAFE)
+    index_path = f"{prefix}{normalized_archive_path}/index.html"
+    normalized_output_path = str(output_path or "").lstrip("#/")
+    if not normalized_output_path:
+        return index_path
+    fragment = quote(normalized_output_path, safe=_URL_FRAGMENT_SAFE)
+    return f"{index_path}#{fragment}"
+
+
+def get_snapshot_output_anchor(plugin: str, output_path: str) -> str:
+    """Map root-relative legacy outputs to the plugin card that displays them."""
+    normalized_output_path = str(output_path or "").lstrip("/")
+    return normalized_output_path if normalized_output_path.startswith(f"{plugin}/") else plugin
+
+
+def build_snapshot_detail_url(
+    archive_path: str,
+    output_path: str = "",
+    request=None,
+    config: dict[str, Any] | None = None,
+    **config_kwargs: Any,
+) -> str:
+    """Build a request-aware URL for trusted snapshot UI on the web origin."""
+    return build_web_url(
+        build_snapshot_detail_path(archive_path, output_path=output_path),
+        request=request,
+        config=config,
+        **config_kwargs,
+    )
+
+
+def build_snapshot_role_output_url(
+    snapshot,
+    role: str,
+    request=None,
+    config: dict[str, Any] | None = None,
+    archive_results=None,
+    fallback_to_default: bool = False,
+    **config_kwargs: Any,
+) -> str:
+    """Build the first available output URL for a plugin-owned snapshot role."""
+    from archivebox.plugins.discovery import get_plugin_default_output_path, get_snapshot_role_names
+
+    for plugin in get_snapshot_role_names(role):
+        output_path = snapshot.plugin_output_path(plugin, archive_results=archive_results)
+        if not output_path and fallback_to_default:
+            output_path = get_plugin_default_output_path(plugin)
+        if output_path:
+            return build_snapshot_url(str(snapshot.id), output_path, request=request, config=config, **config_kwargs)
+    return ""
+
+
+def build_snapshot_files_url(
+    snapshot_id: str,
+    path: str = "",
+    request=None,
+    config: dict[str, Any] | None = None,
+    **config_kwargs: Any,
+) -> str:
+    """Build a request-aware directory-browser URL on the isolated replay origin."""
+    directory_url = build_snapshot_url(snapshot_id, path, request=request, config=config, **config_kwargs)
+    parts = urlsplit(directory_url)
+    normalized_directory_path = f"{parts.path.rstrip('/')}/"
+    directory_url = urlunsplit((parts.scheme, parts.netloc, normalized_directory_path, parts.query, parts.fragment))
+    return _replace_url_query_params(directory_url, files="1")
+
+
+def build_snapshot_zip_url(
+    snapshot_id: str,
+    path: str = "",
+    request=None,
+    config: dict[str, Any] | None = None,
+    **config_kwargs: Any,
+) -> str:
+    """Build a request-aware ZIP download URL for a snapshot directory."""
+    files_url = build_snapshot_files_url(snapshot_id, path, request=request, config=config, **config_kwargs)
+    return _replace_url_query_params(files_url, download="zip")
 
 
 def build_original_url(domain: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_original_base_url(domain, request=request, config=config, **config_kwargs), path)
+    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
+    base_url = _build_url(get_web_base_url(request=request, config=config), f"/original/{domain}")
+    return _build_url(base_url, path)
 
 
 def _build_url(base_url: str, path: str) -> str:
@@ -409,3 +492,11 @@ def _build_url(base_url: str, path: str) -> str:
     if not path:
         return base_url
     return f"{base_url}{path if path.startswith('/') else f'/{path}'}"
+
+
+def _replace_url_query_params(url: str, **params: str) -> str:
+    parts = urlsplit(url)
+    replacement_keys = set(params)
+    query = [(key, value) for key, value in parse_qsl(parts.query, keep_blank_values=True) if key not in replacement_keys]
+    query.extend((key, value) for key, value in params.items())
+    return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment))

archivebox/core/shutdown_util.py

@@ -1,3 +1,28 @@
+"""Shared foreground signal and child-process cleanup contracts.
+
+Shutdown intent and process exit are separate: takeover must preserve the
+replacement owner's work, while a user abort must not later resume the crawl.
+Callers own that policy; this module preserves the received signal and stops
+only the process objects they explicitly supply.
+
+Cooperative boundaries are essential, not just quieter exception handling.
+A real Linux CI regression interrupted Popen.poll() with SIGINT and stranded
+its wait lock. SIGKILL stopped the child (it became a zombie), but Popen.wait()
+still timed out. Raising a different exception or waiting longer cannot repair
+that interrupted critical section. Polling loops must record the signal first
+and honor it between operations; async owners may cancel their own task while
+their loop is running. Once exiting, the last-resort process cleanup must stay
+synchronous: the original loop may be closed, and creating a replacement loop
+cannot recover tasks or resources owned by it.
+
+Keep immediate signal notices and repeated-signal force exit independent of
+the event bus: it may already be draining or closed during shutdown. Resumable
+pause/retry/skip belongs to the existing crawl controller and must not set
+sticky command-shutdown state. See foreground_shutdown_signals below for both
+cooperative and legacy blocking callers; do not change all callers' policy at
+once without supplying their interruption boundaries.
+"""
+
 from __future__ import annotations
 
 import os
@@ -23,7 +48,7 @@ _active_shutdown_state: ShutdownSignalState | None = None
 
 
 def raise_if_shutdown_requested() -> None:
-    """Let long foreground loops honor a signal even if Python ignored it once."""
+    """Honor sticky shutdown intent at a caller-owned interruption boundary."""
 
     if _active_shutdown_state and _active_shutdown_state.signal_name:
         raise KeyboardInterrupt
@@ -48,14 +73,37 @@ def wait_popen_and_kill_children(
     timeout: float,
     kill_timeout: float = 2.0,
 ) -> None:
-    """Wait for a Popen parent and then hard-kill any surviving descendants."""
+    """Reap our own Popen child and stop its explicitly captured descendants.
+
+    A dead child still needs its parent to reap it. Keep wait failures visible;
+    bypassing Popen's lock would hide the interrupted-owner bug described above.
+    Descendants must still be cleaned up when that wait fails.
+    """
 
     try:
-        proc.wait(timeout=timeout)
-    except subprocess.TimeoutExpired:
-        proc.kill()
-        proc.wait(timeout=kill_timeout)
-    kill_remaining_processes(children, timeout=kill_timeout)
+        try:
+            proc.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.wait(timeout=kill_timeout)
+    except subprocess.TimeoutExpired as err:
+        try:
+            state = psutil.Process(proc.pid).as_dict(attrs=["pid", "ppid", "status", "num_threads", "cpu_times"])
+        except psutil.NoSuchProcess:
+            state = {"pid": proc.pid, "status": "exited"}
+        err.add_note(f"Process state after SIGKILL: {state}")
+        # A locked Popen wait can time out even after the OS process exits.
+        # Inspect CPython's lock only for diagnostics; never release it here.
+        waitpid_lock = getattr(proc, "_waitpid_lock", None)
+        err.add_note(
+            f"Popen state: returncode={proc.returncode}, "
+            f"waitpid_lock_locked={waitpid_lock.locked() if waitpid_lock is not None else None}, "
+            f"threads={[thread.name for thread in threading.enumerate()]}",
+        )
+        raise
+    finally:
+        # A parent that cannot be reaped must not leave its descendants running.
+        kill_remaining_processes(children, timeout=kill_timeout)
 
 
 def wait_psutil_and_kill_children(
@@ -101,13 +149,17 @@ def foreground_shutdown_signals(
     *,
     first_signal_message: str | None = "\n[🛑] Got {signal_name}, stopping gracefully...\n",
     on_signal: Callable[[signal.Signals], None] | None = None,
+    interrupt_handlers: dict[signal.Signals, Callable[[], None]] | None = None,
     raise_on_first_signal: bool = True,
 ) -> Iterator[ShutdownSignalState]:
-    """Install foreground signal handlers that print an immediate exit notice.
+    """Keep shutdown intent visible until the owning foreground command unwinds.
 
-    Some log-tail loops intentionally swallow KeyboardInterrupt so that callers
-    can centralize cleanup in finally blocks. The handler writes the signal name
-    immediately, then raises KeyboardInterrupt to break out of the blocking read.
+    Signal state was added because inner loops can swallow KeyboardInterrupt
+    while their caller still needs to stop its children. Cooperative callers use
+    raise_on_first_signal=False and check raise_if_shutdown_requested() between
+    operations, or cancel their owned async task through on_signal. Raising from
+    the handler can otherwise interrupt library locks or partially sent I/O.
+    Blocking callers without a cooperative boundary retain immediate exceptions.
     """
 
     global _active_shutdown_state
@@ -118,10 +170,20 @@ def foreground_shutdown_signals(
 
     state = ShutdownSignalState()
     previous_active_state = _active_shutdown_state
-    previous_handlers = {sig: signal.getsignal(sig) for sig in handled_signals}
+    previous_handlers = {sig: signal.getsignal(sig) for sig in (*handled_signals, *(interrupt_handlers or {}))}
 
     def raise_keyboard_interrupt(signum, _frame):
         sig = signal.Signals(signum)
+        if interrupt_handlers and sig in interrupt_handlers:
+            # Shield resumable terminal intent from sticky shutdown state.
+            # Ctrl+C is delegated to abx-dl's pause/abort controller; SIGUSR1/2
+            # carry prompt answers. Recording any of these as shutdown here
+            # makes a later skip/retry fail raise_if_shutdown_requested(), or
+            # makes a second Ctrl+C hard-exit before hooks can flush output.
+            # SIGTERM/SIGHUP deliberately bypass this shield: ownership takeover
+            # and parent loss must cancel the old worker without asking for input.
+            interrupt_handlers[sig]()
+            return
         already_requested = state.signal_name is not None
         if not already_requested:
             state.signal_name = sig.name
@@ -129,11 +191,9 @@ def foreground_shutdown_signals(
                 os.write(sys.stdout.fileno(), first_signal_message.format(signal_name=state.signal_name).encode())
         if on_signal is not None:
             on_signal(sig)
-        # Foreground `archivebox add` uses the first signal to abort the active
-        # hook through the bus-facing runner code, then reserves the second
-        # signal for hard foreground-command shutdown. Server/update/run and
-        # other non-interactive commands raise immediately so their finally
-        # blocks can stop owned children without prompting.
+        # Actual shutdown requests remain sticky and a second signal forces
+        # exit. Resumable interactive actions were handled above without
+        # marking this context as shutting down.
         if already_requested:
             os.write(sys.stdout.fileno(), f"\n[🛑] Got {sig.name} again, exiting immediately.\n".encode())
             os._exit(130)
@@ -142,7 +202,7 @@ def foreground_shutdown_signals(
 
     try:
         _active_shutdown_state = state
-        for sig in handled_signals:
+        for sig in previous_handlers:
             signal.signal(sig, raise_keyboard_interrupt)
         yield state
     finally:

archivebox/core/snapshot_status.py

@@ -6,7 +6,7 @@ from django.db.models import QuerySet
 def snapshot_status_values() -> tuple[str, ...]:
     from archivebox.core.models import Snapshot
 
-    return tuple(Snapshot.StatusChoices.values)
+    return tuple(value for value, _label in Snapshot._meta.get_field("status").choices)
 
 
 def normalize_snapshot_status(status: str | None) -> str | None:

archivebox/core/tag_util.py

@@ -6,15 +6,16 @@ from typing import Any
 from urllib.parse import unquote
 
 from django.contrib.auth.models import User
-from django.db.models import Count, Exists, F, OuterRef, QuerySet
+from django.db.models import Count, Exists, F, OuterRef, Prefetch, QuerySet
 from django.db.models.functions import Lower
 from django.http import HttpRequest
 from django.urls import reverse
 
 from archivebox.config.common import get_config
+from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag, Tag
+from archivebox.core.routes_util import build_snapshot_detail_url, build_snapshot_role_output_url
 from archivebox.misc.util import sanitize_html_text
-from archivebox.core.routes_util import build_snapshot_url, build_web_url
-from archivebox.core.models import Snapshot, SnapshotTag, Tag
+from archivebox.plugins.discovery import get_snapshot_role_names
 
 
 TAG_SNAPSHOT_PREVIEW_LIMIT = 10
@@ -202,9 +203,16 @@ def _build_snapshot_preview(snapshot: Snapshot, request: HttpRequest | None = No
         "id": str(snapshot.pk),
         "title": _display_snapshot_title(snapshot),
         "url": snapshot.url,
-        "favicon_url": build_snapshot_url(str(snapshot.pk), "favicon.ico", request=request, config=config),
+        "favicon_url": build_snapshot_role_output_url(
+            snapshot,
+            "list_icon",
+            request=request,
+            config=config,
+            archive_results=snapshot._list_icon_results,
+            fallback_to_default=True,
+        ),
         "admin_url": reverse("admin:core_snapshot_change", args=[snapshot.pk]),
-        "archive_url": build_web_url(f"/{snapshot.archive_path_from_db}/index.html", request=request, config=config),
+        "archive_url": build_snapshot_detail_url(snapshot.archive_path_from_db, request=request, config=config),
         "downloaded_at": snapshot.downloaded_at.isoformat() if snapshot.downloaded_at else None,
     }
 
@@ -221,6 +229,22 @@ def _build_snapshot_preview_map(
     snapshot_tags = (
         SnapshotTag.objects.filter(tag_id__in=tag_ids)
         .select_related("snapshot__crawl__created_by")
+        .prefetch_related(
+            Prefetch(
+                "snapshot__archiveresult_set",
+                queryset=ArchiveResult.objects.filter(
+                    plugin__in=get_snapshot_role_names("list_icon"),
+                    status=ArchiveResult.StatusChoices.SUCCEEDED,
+                ).only(
+                    "snapshot_id",
+                    "plugin",
+                    "status",
+                    "output_str",
+                    "output_files",
+                ),
+                to_attr="_list_icon_results",
+            ),
+        )
         .order_by(
             "tag_id",
             F("snapshot__downloaded_at").desc(nulls_last=True),

archivebox/core/takeover_util.py

@@ -62,7 +62,7 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
         # not keep stealing leadership from the next foreground command.
         base_qs.filter(process_type=Process.TypeChoices.ORCHESTRATOR).exclude(parent__process_type=Process.TypeChoices.SUPERVISORD),
     ):
-        for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+        for proc in qs.order_by("-created_at", "-modified_at").paged_iterator(chunk_size=50):
             if proc.is_running:
                 return proc
             proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
@@ -76,6 +76,8 @@ def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:
 
 def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
     """Return the newest live local parent allowed to borrow runner/sonic."""
+    from django.db.models import Q
+
     from archivebox.machine.models import Machine, Process
 
     machine = Machine.current()
@@ -83,11 +85,17 @@ def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
         machine=machine,
         status=Process.StatusChoices.RUNNING,
         pwd=str(data_dir),
-        process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ADD, Process.TypeChoices.UPDATE),
+    ).filter(
+        Q(process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ADD, Process.TypeChoices.UPDATE))
+        # A direct `run` owns its own hooks, just as add/update own theirs via
+        # a supervised one-shot worker. Respawning the server daemon must not
+        # steal the gate back from that newer foreground command. Supervised
+        # runner children cannot elect themselves above their parent command.
+        | (Q(process_type=Process.TypeChoices.ORCHESTRATOR) & ~Q(parent__process_type=Process.TypeChoices.SUPERVISORD)),
     )
     if exclude_id is not None:
         qs = qs.exclude(id=exclude_id)
-    for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+    for proc in qs.order_by("-created_at", "-modified_at").paged_iterator(chunk_size=50):
         if proc.is_running:
             return proc
         proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
@@ -170,7 +178,7 @@ def live_runner_processes(*, data_dir: str | Path):
     qs = qs.filter(machine=machine)
     live = []
     foreign_namespace_ids = []
-    for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20):
+    for process in qs.order_by("started_at", "created_at").paged_iterator(chunk_size=20):
         if not process.shares_pid_namespace:
             foreign_namespace_ids.append(process.id)
             continue

archivebox/core/templatetags/core_tags.py

@@ -1,21 +1,26 @@
+import hashlib
 import logging
+import math
 import os
+import re
 from html import unescape
 from pathlib import Path
 from typing import Any
 from urllib.parse import quote, urlparse
 
-from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target
 from django import template
-from django.urls import reverse
 from django.utils import timezone
-from django.utils.html import escape
+from django.utils.html import escape, format_html
 from django.utils.safestring import mark_safe
 from django.utils.text import Truncator
 
 from archivebox.config import CONSTANTS
 from archivebox.core.routes_util import (
+    build_snapshot_detail_url,
+    build_snapshot_files_url,
+    build_snapshot_role_output_url,
     build_snapshot_url,
+    build_snapshot_zip_url,
     get_admin_base_url,
     get_snapshot_base_url,
     get_web_base_url,
@@ -26,9 +31,46 @@ from archivebox.plugins.discovery import (
     get_plugin_icon,
     get_plugin_name,
     get_plugin_template,
+    plugin_card_is_interactive,
 )
 
 register = template.Library()
+register.filter("plugin_card_is_interactive", plugin_card_is_interactive)
+
+# Preview cards are clipped windows onto their content, never scroll areas.
+# Apply inside the document too: parent overflow cannot constrain an iframe.
+CARD_OVERFLOW_STYLE = (
+    "<style>html,body{height:100%!important;min-height:0!important}"
+    "html,body,body *{overflow:clip!important;scrollbar-width:none!important}</style>"
+)
+
+
+@register.filter
+def snapshot_url_text(url):
+    """Emphasize the hostname while preserving the exact selectable URL."""
+    url = str(url or "")
+    match = re.match(r"^(?:[a-zA-Z][a-zA-Z0-9+.-]*:)?//([^/?#]*)", url)
+    if not match:
+        return escape(url)
+    netloc = match[1]
+    authority = netloc.rsplit("@", 1)[-1]
+    start = match.start(1) + len(netloc) - len(authority)
+    host_end = authority.find("]") + 1 if authority.startswith("[") else authority.find(":")
+    hostname = authority[:host_end] if host_end > 0 else authority
+    if hostname.lower().startswith("www."):
+        start += 4
+        hostname = hostname[4:]
+    remainder = url[start + len(hostname) :]
+    suffix = re.search(r"[?&#]", remainder)
+    suffix_start = suffix.start() if suffix else len(remainder)
+    return format_html(
+        '<span class="snapshot-url-prefix">{}</span><span class="snapshot-url-domain">{}</span>'
+        '<span class="snapshot-url-path">{}</span><span class="snapshot-url-suffix">{}</span>',
+        url[:start],
+        hostname,
+        remainder[:suffix_start],
+        remainder[suffix_start:],
+    )
 
 
 @register.simple_tag(takes_context=True)
@@ -191,7 +233,13 @@ def _list_media_files(result, *, include_filesystem_fallback=True) -> list[dict]
                 candidates.append((rel_path, size))
 
     for rel_path, size in candidates:
-        href = str(Path(result.plugin) / rel_path)
+        if callable(getattr(result, "output_file_path", None)):
+            href = result.output_file_path(str(rel_path), output_file_map=output_files)
+        else:
+            metadata = output_files.get(str(rel_path)) or {}
+            href = str(rel_path) if metadata.get("root_relative") else str(Path(result.plugin) / rel_path)
+        if not href:
+            continue
         suffix = rel_path.suffix.lower()
         media_type = "video" if suffix in _VIDEO_FILE_EXTS else "audio"
         media_files.append(
@@ -270,12 +318,14 @@ def _snapshot_base_url_for_context(context, snapshot) -> str:
 
 def _snapshot_url_for_context(context, snapshot, path: str = "") -> str:
     if not context.get("STATIC_EXPORT"):
-        return build_snapshot_url(
+        path_part, separator, query = str(path or "").partition("?")
+        url = build_snapshot_url(
             str(_snapshot_id(snapshot)),
-            path,
+            path_part,
             request=context.get("request"),
             config=context.get("CONFIG"),
         )
+        return f"{url}?{query}" if separator else url
 
     base_url = _static_snapshot_base_url(context, snapshot)
     raw_path = str(path or "")
@@ -287,14 +337,6 @@ def _snapshot_url_for_context(context, snapshot, path: str = "") -> str:
     return f"{base_url.rstrip('/')}/{quoted_path}{suffix}"
 
 
-def _build_snapshot_files_url(snapshot_id: str, request=None, config=None, base_url: str | None = None) -> str:
-    return (
-        f"{base_url.rstrip('/')}/index.jsonl"
-        if base_url
-        else build_snapshot_url(str(snapshot_id), "/?files=1", request=request, config=config)
-    )
-
-
 def _build_snapshot_preview_url(
     snapshot_id: str,
     path: str = "",
@@ -306,20 +348,25 @@ def _build_snapshot_preview_url(
     if path == "about:blank":
         return path
     if _is_root_snapshot_output_path(path):
-        return _build_snapshot_files_url(snapshot_id, request=request, config=config, base_url=base_url)
+        return (
+            f"{base_url.rstrip('/')}/index.jsonl"
+            if base_url
+            else build_snapshot_files_url(str(snapshot_id), request=request, config=config)
+        )
     if base_url:
         path_part, separator, query = str(path).lstrip("/").partition("?")
         url = f"{base_url.rstrip('/')}/{quote(path_part, safe=_STATIC_URL_SAFE)}"
         if separator:
             url = f"{url}?{query}"
     else:
-        url = build_snapshot_url(str(snapshot_id), path, request=request, config=config)
+        path_part, separator, query = str(path).partition("?")
+        url = build_snapshot_url(str(snapshot_id), path_part, request=request, config=config)
+        if separator:
+            url = f"{url}?{query}"
     path_parts = Path(path).parts
     plugin = get_plugin_name(plugin) if plugin else (path_parts[0] if len(path_parts) > 1 else "")
     has_plugin_preview = bool(plugin and get_plugin_template(plugin, "full", fallback=False))
-    if not (
-        _is_text_preview_path(path) or _is_image_preview_path(path) or has_plugin_preview or is_archivewebpage_replay_target(path or "")
-    ):
+    if not (_is_text_preview_path(path) or _is_image_preview_path(path) or has_plugin_preview):
         return url
     separator = "&" if "?" in url else "?"
     return f"{url}{separator}preview=1"
@@ -385,12 +432,29 @@ def index(value, position):
 
 
 @register.filter
-def file_size(num_bytes: float) -> str:
+def file_size(num_bytes: float, decimal_places: int = 1) -> str:
     for count in ["Bytes", "KB", "MB", "GB"]:
         if num_bytes > -1024.0 and num_bytes < 1024.0:
-            return f"{num_bytes:3.1f} {count}"
+            if num_bytes >= 0:
+                precision = 10 ** max(0, int(decimal_places))
+                num_bytes = math.ceil(num_bytes * precision) / precision
+            return f"{num_bytes:.{decimal_places}f} {count}"
         num_bytes /= 1024.0
-    return "{:3.1f} {}".format(num_bytes, "TB")
+    if num_bytes >= 0:
+        precision = 10 ** max(0, int(decimal_places))
+        num_bytes = math.ceil(num_bytes * precision) / precision
+    return f"{num_bytes:.{decimal_places}f} TB"
+
+
+@register.filter
+def grid_file_size(num_bytes: float | None) -> str:
+    if not num_bytes or num_bytes <= 0:
+        return ""
+    if num_bytes < 1024**2:
+        return f"{max(1, num_bytes / 1024):.0f}kb"
+    if num_bytes < 1024**3:
+        return f"{num_bytes / 1024**2:.0f}mb"
+    return f"{num_bytes / 1024**3:.1f}gb"
 
 
 @register.filter
@@ -557,87 +621,140 @@ def snapshot_base_url(context, snapshot) -> str:
     return _snapshot_base_url_for_context(context, snapshot)
 
 
+@register.simple_tag(takes_context=True)
+def snapshot_detail_url(context, snapshot) -> str:
+    """Keep the trusted snapshot detail page on web.*, not the replay host."""
+    if context.get("STATIC_EXPORT"):
+        return _snapshot_url_for_context(context, snapshot, "index.html")
+    return build_snapshot_detail_url(
+        snapshot.archive_path_from_db,
+        request=context.get("request"),
+        config=context.get("CONFIG"),
+    )
+
+
 @register.simple_tag(takes_context=True)
 def snapshot_url(context, snapshot, path: str = "") -> str:
     return _snapshot_url_for_context(context, snapshot, path)
 
 
 @register.simple_tag(takes_context=True)
-def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> str:
-    snapshot_id = str(_snapshot_id(snapshot))
-    url_cache = snapshot.__dict__.setdefault("_snapshot_archiveresult_url_cache", {})
+def snapshot_files_url(context, snapshot, path: str = "") -> str:
+    if context.get("STATIC_EXPORT"):
+        return _snapshot_url_for_context(context, snapshot, "index.jsonl")
+    return build_snapshot_files_url(
+        str(_snapshot_id(snapshot)),
+        path,
+        request=context.get("request"),
+        config=context.get("CONFIG"),
+    )
+
+
+@register.simple_tag(takes_context=True)
+def snapshot_zip_url(context, snapshot, path: str = "") -> str:
+    if context.get("STATIC_EXPORT"):
+        return ""
+    return build_snapshot_zip_url(
+        str(_snapshot_id(snapshot)),
+        path,
+        request=context.get("request"),
+        config=context.get("CONFIG"),
+    )
+
+
+@register.simple_tag(takes_context=True)
+def snapshot_role_output_url(context, snapshot, role: str, fallback_to_default: bool = False) -> str:
+    url_cache = snapshot.__dict__.setdefault("_snapshot_role_output_url_cache", {})
     cache_key = (
-        plugin,
-        filename,
+        role,
         bool(context.get("STATIC_EXPORT")),
         str(context.get("STATIC_EXPORT_DIR") or ""),
+        fallback_to_default,
     )
     if cache_key in url_cache:
         return url_cache[cache_key]
 
-    results = None
-    if "_admin_archiveresults" in snapshot.__dict__:
-        results = snapshot.__dict__["_admin_archiveresults"]
-    elif "archiveresult_set" in snapshot.__dict__.get("_prefetched_objects_cache", {}):
-        results = snapshot.archiveresult_set.all()
-    elif "_snapshot_archiveresult_url_results" in snapshot.__dict__:
-        results = snapshot.__dict__["_snapshot_archiveresult_url_results"]
+    from archivebox.plugins.discovery import get_snapshot_role_names
 
-    if results is None:
-        from archivebox.core.models import ArchiveResult
-
-        results = list(
-            ArchiveResult.objects.filter(
-                snapshot_id=snapshot_id,
-                plugin__in=("screenshot", "chrome_extension_screenshot", "favicon"),
-                status=ArchiveResult.StatusChoices.SUCCEEDED,
-            ).only(
-                "plugin",
-                "status",
-                "output_files",
-            ),
+    if context.get("STATIC_EXPORT"):
+        output_path = None
+        for plugin in get_snapshot_role_names(role):
+            output_path = snapshot.plugin_output_path(plugin)
+            if output_path:
+                break
+        url_cache[cache_key] = _snapshot_url_for_context(context, snapshot, output_path) if output_path else ""
+    else:
+        url_cache[cache_key] = build_snapshot_role_output_url(
+            snapshot,
+            role,
+            request=context.get("request"),
+            config=context.get("CONFIG"),
+            fallback_to_default=fallback_to_default,
         )
-        snapshot.__dict__["_snapshot_archiveresult_url_results"] = results
-
-    for result in results:
-        if result.plugin != plugin or result.status != "succeeded":
-            continue
-        output_files = result.output_files or {}
-        file_info = output_files.get(filename)
-        if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
-            continue
-        output_path = filename if file_info.get("root_relative") else f"{plugin}/{filename}"
-        url_cache[cache_key] = _snapshot_url_for_context(context, snapshot, output_path)
-        return url_cache[cache_key]
-
-    url_cache[cache_key] = ""
-    return ""
+    return url_cache[cache_key]
 
 
 @register.simple_tag(takes_context=True)
-def admin_snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> str:
-    if not snapshot_archiveresult_url(context, snapshot, plugin, filename):
+def snapshot_result_file_url(context, snapshot, result, path: str) -> str:
+    """Build a saved-file URL from one ArchiveResult declaration."""
+    try:
+        output_path = result.output_file_path(path)
+    except (AttributeError, TypeError, ValueError, OverflowError):
         return ""
-    return reverse("admin:core_snapshot_preview", args=(snapshot.pk, plugin, filename))
+    if not output_path:
+        return ""
+    return _snapshot_url_for_context(context, snapshot, output_path)
 
 
 @register.simple_tag(takes_context=True)
 def snapshot_thumbnail(context, snapshot, admin=False, width="100px", height="100px"):
-    from archivebox.core.preview_util import render_snapshot_preview
+    from django.template.loader import render_to_string
 
-    if admin:
-        url_for_candidate = lambda candidate: reverse(
-            "admin:core_snapshot_preview",
-            args=(snapshot.pk, candidate["plugin"], candidate["filename"]),
+    from archivebox.plugins.output_groups import snapshot_thumbnail_result
+
+    results = snapshot.__dict__.get("_snapshot_card_results")
+    if results is None:
+        results = snapshot.__dict__.get("_admin_output_results")
+    if results is None:
+        results = snapshot.__dict__.get("_admin_archiveresults")
+    if results is None:
+        results = snapshot.__dict__.get("_prefetched_objects_cache", {}).get("archiveresult_set")
+    if results is None:
+        results = snapshot.archiveresult_set.exclude(plugin="").only(
+            "id",
+            "snapshot_id",
+            "plugin",
+            "status",
+            "output_str",
+            "output_files",
+            "output_size",
         )
-    else:
-        url_for_candidate = lambda candidate: _snapshot_url_for_context(context, snapshot, candidate["path"])
-    return render_snapshot_preview(snapshot, url_for_candidate, width=width, height=height)
+
+    try:
+        host = urlparse(snapshot.url).hostname or "Archive"
+    except ValueError:
+        host = "Archive"
+    host = host.removeprefix("www.")
+    hue = int(hashlib.sha256(host.encode()).hexdigest()[:6], 16) % 360
+    return mark_safe(
+        render_to_string(
+            "core/snapshot_thumbnail.html",
+            {
+                **context.flatten(),
+                "card_result": snapshot_thumbnail_result(results),
+                "host": host,
+                "initial": host[:1].upper(),
+                "hue": hue,
+                "width": width,
+                "height": height,
+            },
+        ),
+    )
 
 
 @register.simple_tag(takes_context=True)
 def snapshot_index_row(context, link) -> str:
-    snapshot_base = _snapshot_base_url_for_context(context, link)
+    detail_url = snapshot_detail_url(context, link)
 
     status = getattr(link, "status", None) or "unknown"
     bookmarked_at = getattr(link, "bookmarked_at", None)
@@ -658,7 +775,9 @@ def snapshot_index_row(context, link) -> str:
     title = unescape(getattr(link, "title", "") or "")
     title_text = title or url
     tags_str = link.tags_str() if callable(getattr(link, "tags_str", None)) else getattr(link, "tags_str", "")
-    tag_html = "".join(f'<span class="snapshot-tag">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag)
+    tag_html = "".join(
+        f'<span class="snapshot-tag" title="{escape(tag)}">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag
+    )
     if tag_html:
         tag_cell = f'<span class="snapshot-tags">{tag_html}</span>'
     else:
@@ -673,14 +792,14 @@ def snapshot_index_row(context, link) -> str:
     archive_size = int(getattr(link, "archive_size", 0) or 0)
     size_cell = file_size(archive_size) if archive_size else '<span class="empty-value">...</span>'
     output_plural = "" if num_outputs == 1 else "s"
-    files_url = _snapshot_url_for_context(context, link, "index.jsonl") if context.get("STATIC_EXPORT") else f"{snapshot_base}/?files=1"
+    files_url = snapshot_files_url(context, link)
 
     preview_html = snapshot_thumbnail(context, link)
 
-    if "_public_favicon_paths" in link.__dict__:
-        favicon_paths = list(getattr(link, "_public_favicon_paths", []) or [])
-        if favicon_paths:
-            favicon_urls = [_snapshot_url_for_context(context, link, path) for path in favicon_paths]
+    if "_public_list_icon_paths" in link.__dict__:
+        list_icon_paths = list(getattr(link, "_public_list_icon_paths", []) or [])
+        if list_icon_paths:
+            favicon_urls = [_snapshot_url_for_context(context, link, path) for path in list_icon_paths]
             favicon_html = (
                 f'<img src="{escape(favicon_urls[0])}" '
                 f'data-fallbacks="{escape(",".join(favicon_urls[1:]))}" '
@@ -690,7 +809,7 @@ def snapshot_index_row(context, link) -> str:
         else:
             favicon_html = '<span class="link-favicon link-favicon-empty" aria-hidden="true"></span>'
     else:
-        favicon_url = snapshot_archiveresult_url(context, link, "favicon", "favicon.ico")
+        favicon_url = snapshot_role_output_url(context, link, "list_icon")
         favicon_html = (
             (
                 f'<img src="{escape(favicon_url)}" '
@@ -705,27 +824,24 @@ def snapshot_index_row(context, link) -> str:
     html = f"""
 <tr class="snapshot-row status-{escape(status)}">
     <td class="snapshot-time" title="{escape(title_time)}" data-sort="{escape(sort_value)}">
-        <a href="{escape(snapshot_base)}/index.html">
+        <a href="{escape(detail_url)}">
             <span>{escape(date_text)}</span>
             <small>{escape(time_text)}</small>
         </a>
     </td>
     <td class="snapshot-preview-cell">
-        <a href="{escape(snapshot_base)}/index.html" title="Open archived snapshot">
+        <a href="{escape(detail_url)}" title="Open archived snapshot">
             {preview_html}
         </a>
     </td>
     <td class="snapshot-title-cell" title="{escape(title or url)}">
         <div class="snapshot-title-line">
-            <a href="{escape(snapshot_base)}/index.html" class="snapshot-favicon-link" title="Open archived snapshot">
-                {favicon_html}
-            </a>
-            <a href="{escape(snapshot_base)}/index.html" class="snapshot-title">
-                {escape(Truncator(title_text).chars(110))}
+            <a href="{escape(detail_url)}" class="snapshot-title">
+                {escape(Truncator(title_text).chars(110) if title_text != url else url)}
             </a>
         </div>
         <a href="{escape(url)}" class="snapshot-url" title="{escape(url)}" target="_blank" rel="noopener noreferrer">
-            {escape(url)}
+            {favicon_html}<span class="snapshot-url-text">{snapshot_url_text(url)}</span>
         </a>
         <span class="snapshot-mobile-saved">Saved {escape(date_text)} at {escape(time_text)}</span>
     </td>
@@ -796,6 +912,27 @@ def plugin_card(context, result) -> str:
     if result is None or not isinstance(result, ArchiveResult):
         return ""
 
+    plugin = get_plugin_name(result.plugin)
+    if not context.get("STATIC_EXPORT"):
+        # The cover and expanded stack both embed this one card document. Run
+        # plugin code on the snapshot origin so it can read saved files without
+        # exposing raw archives or the admin session to the collection page.
+        card_url = build_snapshot_url(
+            str(_snapshot_id(result.snapshot)),
+            f"_card/{result.id}",
+            request=context.get("request"),
+            config=context.get("CONFIG"),
+        )
+        return mark_safe(
+            f'<iframe src="{escape(card_url)}" title="{escape(plugin)} card" '
+            'loading="lazy" fetchpriority="low" sandbox="allow-scripts allow-same-origin" '
+            'style="width:100% !important;height:100% !important;transform:none !important;border:0"></iframe>',
+        )
+    return render_plugin_card_document(context, result)
+
+
+def render_plugin_card_document(context, result) -> str:
+    """Render one plugin card template for the trusted snapshot-origin frame."""
     plugin = get_plugin_name(result.plugin)
     has_card_template = bool(get_plugin_template(plugin, "card", fallback=False))
     template_str = get_plugin_template(plugin, "card")
@@ -805,12 +942,7 @@ def plugin_card(context, result) -> str:
     output_url = _snapshot_url_for_context(context, result.snapshot, raw_output_path or "")
 
     icon_html = get_plugin_icon(plugin)
-    plugin_lower = (plugin or "").lower()
-    media_files = (
-        _list_media_files(result, include_filesystem_fallback=bool(context.get("STATIC_EXPORT")))
-        if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl")
-        else []
-    )
+    media_files = _list_media_files(result, include_filesystem_fallback=bool(context.get("STATIC_EXPORT")))
     media_file_count = len(media_files)
     if context.get("STATIC_EXPORT") and media_files:
         media_files = [
@@ -859,6 +991,18 @@ def plugin_card(context, result) -> str:
             rendered = tpl.render(ctx)
             # Only return non-empty content (strip whitespace to check)
             if rendered.strip():
+                if rendered.lstrip().lower().startswith("<!doctype html"):
+                    rendered = rendered.replace("</head>", CARD_OVERFLOW_STYLE + "</head>", 1)
+                if context.get("STATIC_EXPORT") and rendered.lstrip().lower().startswith("<!doctype html"):
+                    # Static exports have no _card route. Keep the same card
+                    # document in both stack positions, with relative saved-file
+                    # URLs resolved against the exported index page.
+                    return mark_safe(
+                        f'<iframe srcdoc="{escape(rendered)}" title="{escape(plugin)} card" '
+                        'loading="lazy" sandbox="allow-scripts allow-same-origin" '
+                        'style="width:100% !important;height:100% !important;max-width:100% !important;'
+                        'transform:none !important;border:0"></iframe>',
+                    )
                 return mark_safe(rendered)
     except (template.TemplateSyntaxError, AttributeError, TypeError, ValueError):
         rendered = ""
@@ -944,6 +1088,7 @@ def plugin_full(context, result) -> str:
                 "output_path": output_url,
                 "output_path_raw": raw_output_path,
                 "plugin": plugin,
+                "plugin_icon": get_plugin_icon(plugin),
             },
         )
         rendered = tpl.render(ctx)

archivebox/core/urls.py

@@ -26,15 +26,14 @@ from archivebox.core.views import (
 )
 from archivebox.progressmonitor.views import live_progress_view
 from archivebox.search.views import public_snapshot_search_stream_view
-from archivebox.opencode.views import opencode_proxy_view
+from abx_plugins.plugins.opencode.archivebox.urls import urlpatterns as agent_urlpatterns
 
 urlpatterns = [
     re_path(r"^static/(?P<path>.*)$", serve_static),
     path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}),
     path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}),
     path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"),
-    re_path(r"^admin/agent/?(?=$|opencode)", include("archivebox.opencode.urls")),
-    re_path(r"^(?P<path>assets/.*)$", opencode_proxy_view, name="opencode-assets"),
+    *agent_urlpatterns,
     path("public/search-stream/", public_snapshot_search_stream_view, name="public-search-stream"),
     path("public/", PublicIndexView.as_view(), name="public-index"),
     path("public.html", RedirectView.as_view(url="/public/"), name="public-index-html"),

archivebox/core/views.py

@@ -1,15 +1,13 @@
 __package__ = "archivebox.core"
 
-from archivebox.core.preview_util import PREVIEW_PLUGINS, preview_candidates
-
 import json
 import os
 import posixpath
 from pathlib import Path
 from typing import ClassVar, cast
 from urllib.parse import quote, urlparse
+from uuid import UUID
 
-from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay
 from admin_data_views.typing import ItemContext, SectionData, TableContext
 from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view
 from django import template
@@ -19,7 +17,6 @@ from django.contrib.auth import HASH_SESSION_KEY, SESSION_KEY, get_user_model
 from django.contrib.auth.mixins import UserPassesTestMixin
 from django.contrib.sessions.models import Session
 from django.core import signing
-from django.core.paginator import InvalidPage
 from django.db.models import Case, IntegerField, Q, Value, When
 from django.http import Http404, HttpRequest, HttpResponse, HttpResponseForbidden, QueryDict
 from django.shortcuts import redirect, render
@@ -50,7 +47,6 @@ from archivebox.config.configset import BaseConfigSet
 from archivebox.core.forms import AddLinkForm
 from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
 from archivebox.core.permissions import (
-    PERMISSIONS_PRIVATE,
     PERMISSIONS_PUBLIC,
     PERMISSIONS_UNLISTED,
     can_view_snapshot,
@@ -61,6 +57,8 @@ from archivebox.core.permissions import (
 )
 from archivebox.core.routes_util import (
     build_admin_url,
+    build_snapshot_detail_url,
+    build_snapshot_files_url,
     build_snapshot_url,
     build_web_url,
     get_admin_host,
@@ -71,7 +69,7 @@ from archivebox.core.routes_util import (
 )
 from archivebox.crawls.models import Crawl
 from archivebox.misc.paginators import AcceleratedPaginator
-from archivebox.misc.serve_static import serve_static_with_byterange_support
+from archivebox.misc.serve_static import FAVICON_CACHE_CONTROL, serve_static_with_byterange_support
 from archivebox.misc.util import (
     base_url,
     filter_queryset_by_uuid_substring,
@@ -80,10 +78,19 @@ from archivebox.misc.util import (
     validate_url,
     without_fragment,
 )
-from archivebox.plugins.discovery import get_plugin_catalog, get_plugin_name, get_plugin_template
+from archivebox.plugins.discovery import (
+    get_plugin_catalog,
+    get_plugin_icon,
+    get_plugin_name,
+    get_plugin_template,
+    get_snapshot_role_names,
+    plugin_has_custom_full_response,
+    serve_plugin_replay_asset,
+)
 from archivebox.plugins.forms import get_plugin_config_binary_urls
 from archivebox.plugins.views import get_config_definition_link
 from archivebox.progressmonitor.views import live_progress_view
+from archivebox.progressmonitor.collection import collection_summary
 from archivebox.search.config import (
     get_search_mode,
     get_search_mode_backend,
@@ -115,7 +122,7 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
     if len(lookup) == 12 and "-" not in lookup:
         # Resolve suffixes using the covering primary-key index before fetching
         # wide snapshot rows and joining their crawl/user data.
-        matches = list(Snapshot.objects.filter(id__endswith=lookup).order_by().values_list("id", flat=True))
+        matches = Snapshot.objects.filter(id__endswith=lookup).order_by().values_list("id", flat=True)
         return snapshots.filter(pk__in=matches).order_by("-created_at", "-downloaded_at").first()
 
     try:
@@ -363,10 +370,6 @@ class SnapshotView(View):
 
         # slug is a timestamp
         if slug.replace(".", "").isdigit():
-            # missing trailing slash -> redirect to index
-            if "/" not in path:
-                return redirect(f"{path}/index.html")
-
             try:
                 try:
                     snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
@@ -375,7 +378,11 @@ class SnapshotView(View):
                         return _private_snapshot_auth_redirect(request, snapshot, archivefile or "index.html")
                     canonical_base = snapshot.url_path
                     if canonical_base != snapshot.legacy_archive_path:
-                        target_path = f"/{canonical_base}/{archivefile or 'index.html'}"
+                        target_path = (
+                            build_snapshot_detail_url(snapshot.archive_path_from_db, request=request)
+                            if not archivefile or archivefile == "index.html"
+                            else build_snapshot_url(str(snapshot.id), archivefile, request=request)
+                        )
                         query = request.META.get("QUERY_STRING")
                         if query:
                             target_path = f"{target_path}?{query}"
@@ -390,8 +397,9 @@ class SnapshotView(View):
                             show_indexes=True,
                             is_archive_replay=True,
                         )
-                    elif archivefile == "index.html":
-                        # if they requested snapshot index, serve live rendered template instead of static html
+                    elif archivefile in ("", "index.html"):
+                        # Both the canonical extensionless URL and the legacy
+                        # index.html URL render the live Django snapshot page.
                         response = self.render_live_index(request, snapshot)
                     else:
                         target = build_snapshot_url(str(snapshot.id), archivefile, request=request)
@@ -425,15 +433,15 @@ class SnapshotView(View):
             except Snapshot.MultipleObjectsReturned:
                 snapshot_hrefs = mark_safe("<br/>").join(
                     format_html(
-                        '{} <a href="/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
+                        '{} <a href="{}"><b><code>{}</code></b></a> {} <b>{}</b>',
                         snap.bookmarked_at.strftime("%Y-%m-%d %H:%M:%S"),
-                        snap.archive_path,
+                        build_snapshot_detail_url(snap.archive_path_from_db, request=request),
                         snap.timestamp,
                         snap.url,
                         snap.title_stripped[:64] or "",
                     )
                     for snap in direct_snapshots_queryset(request, Snapshot.objects.filter(timestamp__startswith=slug))
-                    .only("url", "timestamp", "title", "bookmarked_at")
+                    .select_related("crawl__created_by")
                     .order_by("-bookmarked_at")
                 )
                 return HttpResponse(
@@ -457,27 +465,27 @@ class SnapshotView(View):
                         <title>Snapshot Not Found</title>
                         </head><body>
                         <center><br/><br/><br/>
-                        Snapshot <a href="/{}/index.html" target="_top"><b><code>[{}]</code></b></a>: <a href="{}" target="_blank" rel="noreferrer">{}</a><br/>
-                        was queued on {}, but no files have been saved yet in:<br/><b><a href="/{}/" target="_top"><code>{}</code></a><code>/{}</code></b><br/><br/>
+                        Snapshot <a href="{}" target="_top"><b><code>[{}]</code></b></a>: <a href="{}" target="_blank" rel="noreferrer">{}</a><br/>
+                        was queued on {}, but no files have been saved yet in:<br/><b><a href="{}" target="_top"><code>{}</code></a><code>/{}</code></b><br/><br/>
                         It's possible {} during the last capture on {},<br/>or that the archiving process has not completed yet.<br/>
                         <pre><code># run this cmd to finish/retry archiving this Snapshot</code><br/>
                         <code style="user-select: all; color: #333">archivebox update -t timestamp {}</code></pre><br/><br/>
                         <div class="text-align: left; width: 100%; max-width: 400px">
                         <i><b>Next steps:</i></b><br/>
-                        - list all the <a href="/{}/" target="_top">Snapshot files <code>.*</code></a><br/>
-                        - view the <a href="/{}/index.html" target="_top">Snapshot <code>./index.html</code></a><br/>
+                        - list all the <a href="{}" target="_top">Snapshot files <code>.*</code></a><br/>
+                        - view the <a href="{}" target="_top">Snapshot <code>./index.html</code></a><br/>
                         - go to the <a href="/admin/core/snapshot/{}/change/" target="_top">Snapshot admin</a> to edit<br/>
                         - go to the <a href="/admin/core/snapshot/?id__exact={}" target="_top">Snapshot actions</a> to re-archive<br/>
                         - or return to <a href="/" target="_top">the main index...</a></div>
                         </center>
                         </body></html>
                         """,
-                        snapshot.archive_path,
+                        build_snapshot_detail_url(snapshot.archive_path_from_db, request=request),
                         snapshot.timestamp,
                         snapshot.url,
                         snapshot.url,
                         str(snapshot.bookmarked_at).split(".")[0],
-                        snapshot.archive_path,
+                        build_snapshot_files_url(str(snapshot.id), request=request),
                         snapshot.timestamp,
                         archivefile if str(archivefile) != "None" else "",
                         f"the {archivefile} resource could not be fetched"
@@ -485,8 +493,8 @@ class SnapshotView(View):
                         else "the original site was not available",
                         str(snapshot.bookmarked_at).split(".")[0],
                         snapshot.timestamp,
-                        snapshot.archive_path,
-                        snapshot.archive_path,
+                        build_snapshot_files_url(str(snapshot.id), request=request),
+                        build_snapshot_detail_url(snapshot.archive_path_from_db, request=request),
                         snapshot.pk,
                         snapshot.id,
                     ),
@@ -498,17 +506,23 @@ class SnapshotView(View):
         def _resolve_snapshots_for_slug(slug: str):
             # full URLs go straight to the url-only path (fast, indexed)
             if "://" in slug:
-                return SnapshotView.find_snapshots_for_url(slug)
+                return SnapshotView.find_snapshots_for_url(slug, allow_fallback=False)
             # short uuid-shaped slugs (>=8 hex chars after stripping non-hex) try id matching first
             id_qs = SnapshotView.find_snapshots_for_id(slug)
             if id_qs.exists():
                 return id_qs
             return SnapshotView.find_snapshots_for_url(slug)
 
-        snapshots = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path))
+        snapshots = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path)).select_related("crawl__created_by")
         try:
             if "://" in path:
                 snapshot = snapshots.order_by("-bookmarked_at").first()
+                if snapshot is None:
+                    # Fetch the exact match once before trying legacy URL
+                    # fallbacks; an exists() probe followed by the same lookup
+                    # needlessly doubled reads on every normal redirect.
+                    snapshots = direct_snapshots_queryset(request, self.find_snapshots_for_url(path)).select_related("crawl__created_by")
+                    snapshot = snapshots.order_by("-bookmarked_at").first()
                 if snapshot is None:
                     raise Snapshot.DoesNotExist
             else:
@@ -533,15 +547,15 @@ class SnapshotView(View):
         except Snapshot.MultipleObjectsReturned:
             snapshot_hrefs = mark_safe("<br/>").join(
                 format_html(
-                    '{} <code style="font-size: 0.8em">{}</code> <a href="/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
+                    '{} <code style="font-size: 0.8em">{}</code> <a href="{}"><b><code>{}</code></b></a> {} <b>{}</b>',
                     snap.bookmarked_at.strftime("%Y-%m-%d %H:%M:%S"),
                     str(snap.id)[:8],
-                    snap.archive_path,
+                    build_snapshot_detail_url(snap.archive_path_from_db, request=request),
                     snap.timestamp,
                     snap.url,
                     snap.title_stripped[:64] or "",
                 )
-                for snap in snapshots.only("url", "timestamp", "title", "bookmarked_at").order_by("-bookmarked_at")
+                for snap in snapshots.select_related("crawl__created_by").order_by("-bookmarked_at")
             )
             return HttpResponse(
                 format_html(
@@ -554,7 +568,7 @@ class SnapshotView(View):
                 status=404,
             )
 
-        target_path = build_snapshot_url(str(snapshot.id), "index.html", request=request)
+        target_path = build_snapshot_detail_url(snapshot.archive_path_from_db, request=request)
         query = request.META.get("QUERY_STRING")
         if query:
             target_path = f"{target_path}?{query}"
@@ -659,7 +673,9 @@ class SnapshotPathView(View):
         if snapshot_id:
             requested_base = f"{requested_base}/{snapshot_id}"
         if canonical_base != requested_base:
-            target = f"/{canonical_base}/{path or 'index.html'}"
+            target = f"/{canonical_base}"
+            if path:
+                target = f"{target}/{path}"
             query = request.META.get("QUERY_STRING")
             if query:
                 target = f"{target}?{query}"
@@ -705,11 +721,14 @@ def _safe_archive_relpath(path: str) -> str | None:
     return cleaned
 
 
-def _resolve_archiveresult_relpath(snapshot: Snapshot, rel_path: str) -> tuple[str, ArchiveResult | None]:
-    """Resolve plugin-relative output paths through ArchiveResult.output_files."""
+def _resolve_archiveresult_relpath(
+    snapshot: Snapshot,
+    rel_path: str,
+) -> tuple[str, ArchiveResult | None, tuple[str, ...]]:
+    """Resolve a declared output path and its exact historical fallbacks."""
     parts = Path(rel_path).parts
     if len(parts) < 2:
-        return rel_path, None
+        return rel_path, None, ()
 
     plugin = parts[0]
     plugin_relpath = posixpath.join(*parts[1:])
@@ -721,19 +740,38 @@ def _resolve_archiveresult_relpath(snapshot: Snapshot, rel_path: str) -> tuple[s
         ).only("plugin", "output_files"),
     )
     if not results:
-        return rel_path, None
+        from archivebox.plugins.discovery import get_plugin_default_output_path
 
+        fallbacks = (plugin_relpath,) if get_plugin_default_output_path(plugin) == rel_path else ()
+        return rel_path, None, fallbacks
+
+    declared_paths: list[tuple[str, ArchiveResult]] = []
+    historical_fallbacks: list[str] = []
     for result in results:
-        output_files = result.output_files or {}
+        output_files = result.output_file_map()
         for candidate in (plugin_relpath, rel_path):
-            file_info = output_files.get(candidate)
-            if not isinstance(file_info, dict):
-                continue
-            if file_info.get("root_relative"):
-                return candidate, result
-            return rel_path, result
+            output_path = result.output_file_path(candidate, output_file_map=output_files)
+            if output_path and output_path not in {path for path, _result in declared_paths}:
+                declared_paths.append((output_path, result))
 
-    return rel_path, results[0]
+        # Partially migrated 0.7 rows can declare ``singlefile.html`` while
+        # the old file still lives at SNAP_DIR/singlefile.html, without the
+        # later root_relative metadata bit. Keep that one exact declared key as
+        # a fallback after the normal path returns 404. Do not preflight either
+        # path: the static response performs the only happy-path filesystem IO.
+        if plugin_relpath in output_files and plugin_relpath not in historical_fallbacks:
+            historical_fallbacks.append(plugin_relpath)
+
+    if declared_paths:
+        primary_path, primary_result = declared_paths[0]
+        fallbacks = tuple(
+            dict.fromkeys(
+                path for path in (*[path for path, _result in declared_paths[1:]], *historical_fallbacks) if path != primary_path
+            ),
+        )
+        return primary_path, primary_result, fallbacks
+
+    return rel_path, results[0], ()
 
 
 def _plugin_full_preview_response(
@@ -751,17 +789,18 @@ def _plugin_full_preview_response(
     if not plugin:
         return None
 
-    # ReplayWeb.page needs plugin-owned WACZ inspection and service-worker
-    # context, so it remains the one narrow preview exception below.
-    if plugin == "archivewebpage" and archivewebpage_replay.is_replay_target(rel_path):
-        return None
-
     template_str = get_plugin_template(plugin, "full", fallback=False)
     if not template_str:
         return None
+    if plugin_has_custom_full_response(plugin):
+        # The plugin owns path selection and rendering for this full template.
+        # Defer to the file-serving response hook so unrelated outputs from the
+        # same plugin continue through their normal MIME-specific renderer.
+        return None
 
     raw_query = request.GET.copy()
     raw_query.pop("preview", None)
+    raw_query.pop("titlebar", None)
     output_url = request.path
     if raw_query:
         output_url = f"{output_url}?{raw_query.urlencode()}"
@@ -777,24 +816,40 @@ def _plugin_full_preview_response(
                     "snapshot": snapshot,
                     "output_path": output_url,
                     "output_path_raw": rel_path,
-                    "snapshot_details_url": build_web_url(snapshot.get_absolute_url(), request=request),
+                    "snapshot_details_url": build_snapshot_detail_url(snapshot.archive_path_from_db, request=request),
                     "plugin": plugin,
+                    "plugin_icon": get_plugin_icon(plugin),
                     "preview_base": f"{request.path.rsplit('/', 1)[0]}/",
                 },
             ),
         )
     )
+    if request.GET.get("titlebar") == "0":
+        from archivebox.core.templatetags.core_tags import CARD_OVERFLOW_STYLE
+
+        # The card already has a result header and actions. Only its embedded
+        # trusted full viewer drops the duplicate top bar; opening the same
+        # output directly keeps the complete viewer and controls.
+        rendered = rendered.replace("</head>", CARD_OVERFLOW_STYLE + "<style>body > header { display: none !important; }</style></head>", 1)
     response = HttpResponse(rendered, content_type="text/html; charset=utf-8")
     response.headers["Content-Disposition"] = f'inline; filename="{Path(rel_path).stem}.html"'
     response.headers["X-Content-Type-Options"] = "nosniff"
     response.headers["X-ArchiveBox-Security-Mode"] = request.archivebox_config.SERVER_SECURITY_MODE
     response.headers["Referrer-Policy"] = "no-referrer"
-    # Trusted viewers live on the snapshot origin, but the collection UI lives
-    # on web.* (and can be embedded by admin.*). Permit only those configured
-    # origins, not arbitrary sites or other snapshots. This does not grant the
-    # archived document access to the parent frame or its admin session cookies.
+    # Full plugin viewers are trusted code, but must fetch saved data from the
+    # isolated snapshot origin. The outer collection UI stays on web.*.
+    response.headers["Content-Security-Policy"] = _trusted_snapshot_frame_csp(request)
+    return response
+
+
+def _trusted_snapshot_frame_csp(request: HttpRequest) -> str:
+    # A trusted plugin frame on snap-* may be embedded by web.* or admin.*.
+    # Archived HTML/JS/CSS is still served only inside that snapshot origin;
+    # neither this CSP nor the frame grants it admin cookies or cross-snapshot
+    # reads. In one-domain no-JS mode the hosts coincide, but raw replay scripts
+    # remain blocked by the separate replay response policy.
     viewer_origins = " ".join(dict.fromkeys((build_web_url(request=request), build_admin_url(request=request))))
-    response.headers["Content-Security-Policy"] = (
+    return (
         "default-src 'self' data: blob:; "
         "script-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; "
         "style-src 'unsafe-inline' data: blob: 'self'; "
@@ -809,6 +864,47 @@ def _plugin_full_preview_response(
         "form-action 'none'; "
         f"frame-ancestors 'self' {viewer_origins};"
     )
+
+
+def _plugin_card_document_response(request: HttpRequest, snapshot: Snapshot, result_id: str) -> HttpResponse:
+    """Serve the existing card template from the snapshot origin, not web.*.
+
+    srcdoc and inline card scripts inherit the containing page's origin. This
+    frame route gives them same-origin access to this snapshot's saved files
+    without opening CORS on raw archives or sharing the admin session cookie.
+    The parent web page receives only the one-bit login hint; delete buttons
+    navigate to admin.* for a real authenticated confirmation.
+    """
+    try:
+        result_pk = UUID(result_id)
+    except (TypeError, ValueError):
+        raise Http404 from None
+    result = snapshot.archiveresult_set.filter(pk=result_pk).first()
+    if result is None:
+        raise Http404
+
+    # There is exactly one plugin card template. The stack cover and expanded
+    # tray point at this same document; only their containing card width differs.
+    from archivebox.core.templatetags.core_tags import CARD_OVERFLOW_STYLE, render_plugin_card_document
+
+    fragment = render_plugin_card_document(
+        template.Context({"request": request, "CONFIG": get_request_config(request, resolve_plugins=False)}),
+        result,
+    )
+    if str(fragment).lstrip().lower().startswith("<!doctype html"):
+        card_html = str(fragment)
+    else:
+        card_html = (
+            '<!doctype html><html><head><meta charset="utf-8"><style>'
+            "html,body{margin:0;width:100%;height:100%;overflow:hidden}"
+            "body>iframe{display:block;width:100%;height:100%;border:0}"
+            f"</style>{CARD_OVERFLOW_STYLE}</head><body>"
+            f"{fragment}</body></html>"
+        )
+    response = HttpResponse(card_html, content_type="text/html; charset=utf-8")
+    response.headers["Content-Security-Policy"] = _trusted_snapshot_frame_csp(request)
+    response.headers["X-Content-Type-Options"] = "nosniff"
+    response.headers["Referrer-Policy"] = "no-referrer"
     return response
 
 
@@ -927,7 +1023,7 @@ def _build_snapshot_replay_response(request: HttpRequest, snapshot: Snapshot, pa
     snapshot._runtime_config = request_config
 
     if rel_path.startswith("replay/") or rel_path == "replay":
-        response = archivewebpage_replay.serve_replay_asset_response(rel_path, request_config, HttpResponse)
+        response = serve_plugin_replay_asset(rel_path, request_config, HttpResponse)
         if response is not None:
             return response
 
@@ -938,9 +1034,24 @@ def _build_snapshot_replay_response(request: HttpRequest, snapshot: Snapshot, pa
         # subdomain (this keeps the endpoint identical across all security modes).
         return live_progress_view(request, authorized_snapshot=snapshot)
 
+    if rel_path.startswith("_card/"):
+        # SnapshotHostView/SnapshotReplayView have already authorized this
+        # snapshot. Look up the result through its relation so a card ID from
+        # another isolated snapshot can never render here.
+        return _plugin_card_document_response(request, snapshot, rel_path.removeprefix("_card/"))
+
     is_directory_request = bool(path) and path.endswith("/")
     show_indexes = bool(request.GET.get("files")) or (request_config.USES_SUBDOMAIN_ROUTING and is_directory_request)
     if not show_indexes and (not rel_path or rel_path == "index.html"):
+        if request_config.CONTROL_PLANE_ENABLED and snapshot.permissions in (PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED):
+            # Snapshot detail is trusted application UI and belongs on web.*.
+            # snap-* is for isolated saved outputs and plugin frames. Keep the
+            # private replay-cookie path below: its grant is host-only and must
+            # never be mistaken for an admin session on web.*.
+            target = build_snapshot_detail_url(snapshot.archive_path_from_db, request=request, config=request_config)
+            if request.META.get("QUERY_STRING"):
+                target = f"{target}?{request.META['QUERY_STRING']}"
+            return redirect(target)
         return SnapshotView.render_live_index(request, snapshot)
 
     if not rel_path or rel_path.endswith("/"):
@@ -952,22 +1063,23 @@ def _build_snapshot_replay_response(request: HttpRequest, snapshot: Snapshot, pa
     if rel_path is None:
         raise Http404
 
-    rel_path, archive_result = _resolve_archiveresult_relpath(snapshot, rel_path)
+    rel_path, archive_result, fallback_relpaths = _resolve_archiveresult_relpath(snapshot, rel_path)
 
     plugin_preview = _plugin_full_preview_response(request, snapshot, rel_path, archive_result)
     if plugin_preview is not None:
         return plugin_preview
 
-    try:
-        return serve_static_with_byterange_support(
-            request,
-            rel_path,
-            document_root=snapshot.output_dir,
-            show_indexes=show_indexes,
-            is_archive_replay=True,
-        )
-    except Http404:
-        pass
+    for candidate_path in (rel_path, *fallback_relpaths):
+        try:
+            return serve_static_with_byterange_support(
+                request,
+                candidate_path,
+                document_root=snapshot.output_dir,
+                show_indexes=show_indexes,
+                is_archive_replay=True,
+            )
+        except Http404:
+            continue
 
     host = urlparse(snapshot.url).hostname or snapshot.domain
     responses_root = Path(snapshot.output_dir) / "responses" / host
@@ -981,6 +1093,8 @@ def _build_snapshot_replay_response(request: HttpRequest, snapshot: Snapshot, pa
 
 def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""):
     response = _build_snapshot_replay_response(request, snapshot, path)
+    if response.status_code in (200, 206, 304) and response.get("Cache-Control") == FAVICON_CACHE_CONTROL:
+        return response
     if path != "progress.json" and response.status_code in (200, 206, 304):
         if not response.get("Cache-Control"):
             policy = "public" if snapshot.permissions == PERMISSIONS_PUBLIC else "private"
@@ -1104,11 +1218,22 @@ class OriginalDomainReplayView(View):
 
 
 class PublicIndexView(ListView):
+    """Keep browsing responsive while the same small server is archiving.
+
+    Performance takes priority over perfect initial totals: on a 1 vCPU / 1 GB
+    host with hundreds of thousands of snapshots, even an indexed count can
+    consume the entire roughly 0.5-0.7 second page-load budget. Benchmark with
+    the runner active, since it shares the CPU, memory, and SQLite database.
+
+    The unfiltered list therefore renders rows first and accepts missing/stale
+    display totals. A later ordinary reload can show the background-cached
+    count; do not restore a synchronous COUNT or add polling just to fill it in.
+    """
+
     template_name = "public_index.html"
     model = Snapshot
     ordering: ClassVar[list[str]] = ["-bookmarked_at", "-created_at"]
     paginator_class = AcceleratedPaginator
-    public_page_scan_chunk_size = 50
 
     def get_paginate_by(self, queryset):
         runtime_config = self.__dict__.get("runtime_config")
@@ -1129,43 +1254,76 @@ class PublicIndexView(ListView):
             "status",
             "output_size",
             "permissions",
+            # Modern replay URLs need the filesystem version and owner's name.
+            # Deferring these caused four extra SELECTs per displayed snapshot.
+            "fs_version",
+            "crawl__created_by__username",
         )
 
-    def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot] | None:
-        target_count = page_number * page_size
-        public_snapshots: list[Snapshot] = []
-        scanned = 0
-        chunk_size = max(self.public_page_scan_chunk_size, page_size)
-        ordered_snapshots = Snapshot.objects.order_by(*self.ordering).only(*self._base_public_snapshot_fields())
-
-        while len(public_snapshots) < target_count:
-            chunk = list(ordered_snapshots[scanned : scanned + chunk_size])
-            if not chunk:
-                break
-            scanned += len(chunk)
-            public_snapshots.extend(snapshot for snapshot in chunk if snapshot.permissions == PERMISSIONS_PUBLIC)
-
+    def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot]:
+        # Walk the existing chronological index in bounded keyset pages,
+        # reading only identity/visibility until reaching the requested page.
+        # Repeated OFFSET batches used to revisit every earlier row AND join
+        # owners each time, making deep pages quadratic. Filtering visibility
+        # in SQL instead made SQLite sort the entire permissions index first.
+        # Each helper page closes its cursor before yielding; only the final
+        # displayed IDs load wide rows/owners. No new index or SQLite hint.
         start = (page_number - 1) * page_size
-        return public_snapshots[start:target_count]
+        visible_count = 0
+        page_ids = []
+        candidates = Snapshot.objects.only("id", "permissions").order_by(*self.ordering, "id")
+        for snapshot in candidates.paged_iterator(chunk_size=max(100, page_size + 1)):
+            if snapshot.permissions != PERMISSIONS_PUBLIC:
+                continue
+            if visible_count >= start:
+                page_ids.append(snapshot.pk)
+                if len(page_ids) > page_size:
+                    break
+            visible_count += 1
+        return list(
+            Snapshot.objects.filter(pk__in=page_ids, permissions=PERMISSIONS_PUBLIC)
+            .select_related("crawl__created_by")
+            .order_by(*self.ordering, "id")
+            .only(*self._base_public_snapshot_fields()),
+        )
 
     def paginate_queryset(self, queryset, page_size):
         if self.request.GET.get("q", default="").strip():
             return super().paginate_queryset(queryset, page_size)
 
-        public_count = self.get_exact_public_snapshot_count()
-        paginator = self.get_paginator(range(public_count), page_size)
         page_kwarg = self.kwargs.get(self.page_kwarg)
         page_query = self.request.GET.get(self.page_kwarg)
         page_number = page_kwarg or page_query or 1
 
         try:
-            page = paginator.page(page_number)
-        except InvalidPage as err:
+            page_number = int(page_number)
+            if page_number < 1:
+                raise ValueError("Page must be positive")
+        except (ValueError, TypeError) as err:
             raise Http404(f"Invalid page ({page_number}): {err}") from err
 
-        object_list = self._ordered_public_page_from_order_index(page_number=page.number, page_size=page_size)
-        page.object_list = object_list
-        return paginator, page, object_list, page.has_other_pages()
+        snapshots = self._ordered_public_page_from_order_index(page_number=page_number, page_size=page_size)
+        if not snapshots and page_number > 1:
+            raise Http404(f"Invalid page ({page_number}): No snapshots")
+        # Exhausting the rows gives an exact total for this read at no extra
+        # cost. Otherwise the lookahead is only a lower bound: an absent count
+        # is preferable to an on-request scan or a cached total smaller than
+        # the rows we just observed. Cached overestimates are acceptable here.
+        observed_count = (page_number - 1) * page_size + len(snapshots)
+        self.public_snapshot_count = observed_count
+        if len(snapshots) > page_size:
+            summary = collection_summary(self.request.user, public=True)
+            cached_count = summary["snapshots"] if summary is not None else None
+            self.public_snapshot_count = cached_count if cached_count is not None and cached_count >= observed_count else None
+
+        # Give Django an in-memory lower bound, never the collection QuerySet
+        # whose Paginator.count would issue COUNT(*). Navigation uses this read's
+        # lookahead even when a stale total would hide a newly added page.
+        # The template must use the separate display totals, not num_pages here.
+        paginator = self.get_paginator(range(observed_count), page_size)
+        page = paginator.page(page_number)
+        page.object_list = snapshots[:page_size]
+        return paginator, page, page.object_list, page.has_other_pages()
 
     def get_context_data(self, **kwargs):
         runtime_config = self.__dict__.get("runtime_config")
@@ -1194,11 +1352,20 @@ class PublicIndexView(ListView):
             and search_mode_backend
             and context["paginator"].count == 0,
         )
+        context["public_snapshot_count"] = context["paginator"].count if query else self.public_snapshot_count
+        count = context["public_snapshot_count"]
+        context["public_page_count"] = (
+            max(1, (count + context["paginator"].per_page - 1) // context["paginator"].per_page) if count is not None else None
+        )
         snapshots = list(context.get("object_list") or ())
-        icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots}
+        from archivebox.plugins.output_groups import plugin_output_sizes
+
+        all_results_by_snapshot = {str(snapshot.id): [] for snapshot in snapshots}
+        snapshots_by_id = {str(snapshot.id): snapshot for snapshot in snapshots}
+        icons_by_snapshot: dict[str, dict[str, ArchiveResult]] = {str(snapshot.id): {} for snapshot in snapshots}
         tag_names_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
-        preview_results_by_snapshot = {str(snapshot.id): [] for snapshot in snapshots}
-        favicon_paths_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
+        list_icon_paths_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
+        list_icon_plugins = set(get_snapshot_role_names("list_icon"))
         progress_by_snapshot: dict[str, dict[str, int]] = {
             str(snapshot.id): {
                 "total": 0,
@@ -1219,62 +1386,66 @@ class PublicIndexView(ListView):
             ):
                 tag_names_by_snapshot[str(snapshot_id)].append(tag_name)
 
-            for snapshot_id, plugin, status in (
-                ArchiveResult.objects.filter(
-                    snapshot_id__in=icons_by_snapshot.keys(),
-                )
+            results = (
+                ArchiveResult.objects.filter(snapshot_id__in=icons_by_snapshot.keys())
                 .exclude(plugin="")
-                .values_list("snapshot_id", "plugin", "status")
-                .iterator(chunk_size=1000)
-            ):
-                snapshot_key = str(snapshot_id)
+                .only(
+                    "snapshot_id",
+                    "plugin",
+                    "status",
+                    "output_str",
+                    "output_files",
+                    "output_size",
+                )
+            )
+            # This is already bounded by the displayed snapshot page. Finish
+            # the SELECT before parsing output manifests: a streaming cursor
+            # otherwise keeps SQLite's read lock throughout rendering.
+            for result in list(results):
+                snapshot_key = str(result.snapshot_id)
+                # Thumbnail URLs traverse result.snapshot too; share the page's
+                # already-loaded parent rather than fetching it once per result.
+                result.snapshot = snapshots_by_id[snapshot_key]
+                # These read-only page objects are shared by icons, sizes, and
+                # thumbnails. Normalize each manifest once for this render.
+                result._render_output_file_map = result.output_file_map()
+                all_results_by_snapshot[snapshot_key].append(result)
                 progress = progress_by_snapshot[snapshot_key]
                 progress["total"] += 1
-                if status == ArchiveResult.StatusChoices.SUCCEEDED:
-                    icons_by_snapshot[snapshot_key].add(plugin)
+                if result.status == ArchiveResult.StatusChoices.SUCCEEDED:
+                    icons_by_snapshot[snapshot_key].setdefault(result.plugin, result)
                     progress["succeeded"] += 1
-                elif status == ArchiveResult.StatusChoices.FAILED:
+                elif result.status == ArchiveResult.StatusChoices.FAILED:
                     progress["failed"] += 1
-                elif status == ArchiveResult.StatusChoices.STARTED:
+                elif result.status == ArchiveResult.StatusChoices.STARTED:
                     progress["running"] += 1
-                elif status == ArchiveResult.StatusChoices.SKIPPED:
+                elif result.status == ArchiveResult.StatusChoices.SKIPPED:
                     progress["skipped"] += 1
-                elif status == ArchiveResult.StatusChoices.NORESULTS:
+                elif result.status == ArchiveResult.StatusChoices.NORESULTS:
                     progress["noresults"] += 1
-
-            for result in (
-                ArchiveResult.objects.filter(
-                    snapshot_id__in=icons_by_snapshot.keys(),
-                    plugin__in=PREVIEW_PLUGINS,
-                )
-                .only("snapshot_id", "plugin", "status", "output_files")
-                .iterator(chunk_size=1000)
-            ):
-                snapshot_key = str(result.snapshot_id)
-                preview_results_by_snapshot[snapshot_key].append(result)
-                for candidate in preview_candidates([result]):
-                    if candidate["kind"] == "favicon":
-                        favicon_paths_by_snapshot[snapshot_key].append(candidate["path"])
+                if result.plugin in list_icon_plugins and result.status == ArchiveResult.StatusChoices.SUCCEEDED:
+                    if output_path := result.embed_path():
+                        list_icon_paths_by_snapshot[snapshot_key].append(output_path)
 
         for snapshot in snapshots:
             snapshot._icons_compact = True
+            snapshot._icons_output_sizes = plugin_output_sizes(all_results_by_snapshot[str(snapshot.id)])
             snapshot._icons_archive_results = icons_by_snapshot.get(str(snapshot.id), set())
             snapshot._icons_progress_stats = progress_by_snapshot.get(str(snapshot.id), {})
             snapshot.num_outputs_cached = snapshot._icons_progress_stats.get("succeeded", 0)
             snapshot._tags_str_cached = ",".join(tag_names_by_snapshot.get(str(snapshot.id), []))
-            snapshot._preview_results = preview_results_by_snapshot.get(str(snapshot.id), [])
-            snapshot._public_favicon_paths = favicon_paths_by_snapshot.get(str(snapshot.id), [])
+            snapshot._snapshot_card_results = all_results_by_snapshot[str(snapshot.id)]
+            snapshot._public_list_icon_paths = list_icon_paths_by_snapshot.get(str(snapshot.id), [])
             snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
         context["object_list"] = snapshots
         return context
 
-    def get_exact_public_snapshot_count(self) -> int:
-        hidden_count = Snapshot.objects.filter(permissions=PERMISSIONS_PRIVATE).count()
-        hidden_count += Snapshot.objects.filter(permissions=PERMISSIONS_UNLISTED).count()
-        return Snapshot.objects.count() - hidden_count
-
     def get_queryset(self, **kwargs):
-        qs = public_snapshots_queryset(super().get_queryset(**kwargs)).only(*self._base_public_snapshot_fields())
+        qs = (
+            public_snapshots_queryset(super().get_queryset(**kwargs))
+            .select_related("crawl__created_by")
+            .only(*self._base_public_snapshot_fields())
+        )
         query = self.request.GET.get("q", default="").strip()
 
         if not query:
@@ -1553,7 +1724,7 @@ class WebAddView(AddView):
         if requested_url:
             snapshot = self._latest_snapshot_for_url(requested_url)
             if snapshot:
-                return redirect(f"/{snapshot.url_path}")
+                return redirect(build_snapshot_detail_url(snapshot.archive_path_from_db, request=request))
 
         request_host = (request.get_host() or "").lower()
         request_config = get_request_config(request)
@@ -1593,7 +1764,7 @@ class WebAddView(AddView):
 
         snapshot = self._latest_snapshot_for_url(requested_url)
         if snapshot:
-            return redirect(f"/{snapshot.url_path}")
+            return redirect(build_snapshot_detail_url(snapshot.archive_path_from_db, request=request))
 
         add_url = self._normalize_add_url(requested_url)
         assert self.form_class is not None
@@ -1625,7 +1796,7 @@ class WebAddView(AddView):
         crawl = self._create_crawl_from_form(form)
         snapshot = Snapshot.from_json({"url": add_url, "tags": form.cleaned_data.get("tag", "")}, overrides={"crawl": crawl})
         assert snapshot is not None
-        return redirect(f"/{snapshot.url_path}")
+        return redirect(build_snapshot_detail_url(snapshot.archive_path_from_db, request=request))
 
 
 class HealthCheckView(View):

archivebox/core/widgets.py

@@ -717,7 +717,7 @@ class InlineTagEditorWidget(TagEditorWidget):
                 )
             pills_html += f'''
                 <span class="tag-pill" data-tag="{self._escape(td["name"])}" data-tag-id="{td["id"]}" style="{self._tag_style(td["name"])}">
-                    <a href="/admin/core/snapshot/?tags__id__exact={td["id"]}" class="tag-link">{self._escape(td["name"])}</a>
+                    <a href="/admin/core/snapshot/?tags__id__exact={td["id"]}" class="tag-link" title="{self._escape(td["name"])}">{self._escape(td["name"])}</a>
                     {remove_button}
                 </span>
             '''

archivebox/crawls/admin.py

@@ -8,7 +8,7 @@ from urllib.parse import urlencode, urlparse
 from django import forms
 from django.contrib import admin, messages
 from django.core.paginator import Paginator
-from django.db.models import Count, F, Q
+from django.db.models import Count, F, Prefetch, Q
 from django.http import HttpRequest, HttpResponseBadRequest, HttpResponseNotAllowed, JsonResponse
 from django.shortcuts import get_object_or_404, redirect
 from django.template.loader import render_to_string
@@ -30,9 +30,11 @@ from archivebox.core.permissions import (
     PERMISSIONS_VALUES,
     normalize_permissions,
 )
+from archivebox.core.routes_util import build_snapshot_detail_url, build_snapshot_role_output_url
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
 from archivebox.crawls.models import Crawl, CrawlSchedule
 from archivebox.misc.paginators import AcceleratedPaginator
+from archivebox.plugins.discovery import get_snapshot_role_names
 from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.workers.models import RETRY_AT_MAX
 
@@ -72,13 +74,32 @@ def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50,
 
     # Keep ArchiveResult counters as scalar subqueries so the paginated
     # Snapshot queryset does not become a join+GROUP BY over every result row.
-    snapshots_qs = filtered_qs.order_by("-created_at").annotate(
-        total_results=ArchiveResult.snapshot_count_expr(),
-        succeeded_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
-        failed_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.FAILED),
-        started_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.STARTED),
-        skipped_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SKIPPED),
-        snapshot_permissions=F("permissions"),
+    snapshots_qs = (
+        filtered_qs.order_by("-created_at")
+        .annotate(
+            total_results=ArchiveResult.snapshot_count_expr(),
+            succeeded_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
+            failed_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.FAILED),
+            started_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.STARTED),
+            skipped_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SKIPPED),
+            snapshot_permissions=F("permissions"),
+        )
+        .prefetch_related(
+            Prefetch(
+                "archiveresult_set",
+                queryset=ArchiveResult.objects.filter(
+                    plugin__in=get_snapshot_role_names("list_icon"),
+                    status=ArchiveResult.StatusChoices.SUCCEEDED,
+                ).only(
+                    "snapshot_id",
+                    "plugin",
+                    "status",
+                    "output_str",
+                    "output_files",
+                ),
+                to_attr="_list_icon_results",
+            ),
+        )
     )
 
     page_number = request.GET.get(page_param, 1) if request is not None else 1
@@ -219,6 +240,14 @@ def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50,
 
         # Format date
         date_str = snapshot.created_at.strftime("%Y-%m-%d %H:%M") if snapshot.created_at else "-"
+        detail_url = build_snapshot_detail_url(snapshot.archive_path_from_db, request=request)
+        favicon_url = build_snapshot_role_output_url(
+            snapshot,
+            "list_icon",
+            request=request,
+            archive_results=snapshot._list_icon_results,
+            fallback_to_default=True,
+        )
 
         rows.append(f'''
             <tr style="border-bottom: 1px solid #eee;">
@@ -231,8 +260,8 @@ def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50,
                     <span title="{permissions}" style="display:inline-flex; align-items:center; justify-content:center; width:22px; height:22px; border-radius:999px; font-size:12px; color:{permission_fg}; background:{permission_bg};">{permission_icon}</span>
                 </td>
                 <td style="padding: 6px 8px; white-space: nowrap;">
-                    <a href="/{snapshot.archive_path}/" style="text-decoration: none;">
-                        <img src="/{snapshot.archive_path}/favicon.ico"
+                    <a href="{escape(detail_url)}" style="text-decoration: none;">
+                        <img src="{escape(favicon_url)}"
                              style="width: 16px; height: 16px; vertical-align: middle; margin-right: 4px;"
                              onerror="this.style.display='none'"/>
                     </a>
@@ -918,7 +947,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         updated = 0
         batch = []
         crawls_to_update = []
-        for crawl in queryset.only("id", "config", "permissions").iterator(chunk_size=500):
+        for crawl in queryset.only("id", "config", "permissions").order_by("pk").paged_iterator(chunk_size=500):
             old_permissions = crawl.permissions
             config = dict(crawl.config or {})
             config["PERMISSIONS"] = permissions

archivebox/crawls/models.py

@@ -23,6 +23,7 @@ from archivebox.config.common import rprint as print
 from archivebox.core.permissions import PERMISSIONS_VALUES, normalize_permissions
 
 from archivebox.base_models.models import (
+    PagedQuerySet,
     ModelWithUUID,
     ModelWithDeleteAfter,
     ModelWithOutputDir,
@@ -148,6 +149,8 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
 
 
 class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithQueue):
+    objects = PagedQuerySet.as_manager()
+
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
@@ -229,7 +232,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         if paused and save and self.pk:
             from archivebox.core.models import Snapshot
 
-            for snapshot in self.snapshot_set.exclude(status__in=Snapshot.FINAL_STATES).iterator():
+            for snapshot in self.snapshot_set.exclude(status__in=Snapshot.FINAL_STATES).order_by("pk").paged_iterator():
                 snapshot.pause()
         return paused
 
@@ -301,7 +304,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True, config__has_key="DELETE_AFTER")
+        return cls.objects.filter(delete_at__isnull=True, config__has_key="DELETE_AFTER").only("id", "created_at", "delete_at", "config")
 
     def save(self, *args, **kwargs):
         update_fields = kwargs.get("update_fields")
@@ -368,7 +371,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         batch = []
         updated = 0
         queryset = self.snapshot_set.filter(Q(permissions=old_permissions) | Q(permissions__isnull=True)).only("id", "config")
-        for snapshot in queryset.iterator(chunk_size=500):
+        for snapshot in queryset.order_by("pk").paged_iterator(chunk_size=500):
             config = dict(snapshot.config or {})
             config["PERMISSIONS"] = normalized_new_permissions
             snapshot.config = config
@@ -427,7 +430,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=added_names)}
 
             tag_ids = [tag.pk for tag_name in added_names if (tag := tags_by_name.get(tag_name))]
-            snapshot_ids = Snapshot.objects.filter(crawl=self).values_list("id", flat=True).iterator(chunk_size=5000)
+            # Fetch each page before writing M2M rows. iterator(chunk_size=...)
+            # would keep one read cursor open across every bulk write.
+            snapshot_ids = (
+                snapshot.pk for snapshot in Snapshot.objects.filter(crawl=self).only("id").order_by("pk").paged_iterator(chunk_size=500)
+            )
             while True:
                 batch_snapshot_ids = list(islice(snapshot_ids, 5000))
                 if not batch_snapshot_ids:
@@ -846,24 +853,31 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         output_dir: Path | None = None,
         num_snapshots: int | None = None,
     ) -> str:
-        from abx_dl.limits import CrawlLimitState
+        from abx_dl.limits import parse_filesize_to_bytes
 
-        if output_dir is None:
-            output_dir = self.output_dir
         if config is None:
             from archivebox.config.common import get_config
 
-            config = get_config(crawl=self, include_machine=False).for_crawl_runtime(
-                crawl=self,
-                persona=self.resolve_persona(),
-                crawl_output_dir=output_dir,
-            )
+            config = get_config(crawl=self, include_machine=False)
 
-        limits_path = output_dir / ".abx-dl" / "limits.json"
-        if limits_path.exists():
-            stop_reason = CrawlLimitState.from_config(config).get_stop_reason()
-            if stop_reason:
-                return stop_reason
+        # The DB is the shared authority for workers and the UI. A second
+        # ledger beside the artifacts can disagree after cancellation/retry and
+        # would require filesystem access just to render a crawl's status.
+        # Recompute against the current limits so increasing a cap permits work.
+        timeout = int(self._config_value(config, "CRAWL_TIMEOUT", 0) or 0)
+        if timeout:
+            # Snapshot creation already supplies a durable crawl clock. Using
+            # that existing fact avoids another timestamp to keep in sync and
+            # survives cancellation, which deliberately removes unfinished
+            # ArchiveResult rows while preserving the Snapshot for retry.
+            first_snapshot_time = self.snapshot_set.aggregate(first=models.Min("created_at"))["first"]
+            if first_snapshot_time and (timezone.now() - first_snapshot_time).total_seconds() >= timeout:
+                return "crawl_timeout"
+        max_size = parse_filesize_to_bytes(self._config_value(config, "CRAWL_MAX_SIZE", 0) or 0)
+        if max_size:
+            total_size = self.snapshot_set.aggregate(total=models.Sum("output_size"))["total"] or 0
+            if total_size >= max_size:
+                return "crawl_max_size"
 
         max_urls = int(self._config_value(config, "CRAWL_MAX_URLS", 0) or 0)
         if num_snapshots is None:
@@ -1250,10 +1264,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             }
             if tag_names:
                 tag_names_by_url[snapshot.url] = tag_names
-            # Snapshot.save() handles model-level validation. The crawl symlink
-            # can still wait until after commit so SQLite does not hold a write
-            # lock while touching the filesystem.
-            transaction.on_commit(lambda snapshot=snapshot: snapshot.ensure_crawl_symlink())
 
         tag_names = {tag for tags in tag_names_by_url.values() for tag in tags}
         if tag_names:

archivebox/machine/detect.py

@@ -57,7 +57,10 @@ def get_vm_info():
 
     hw_manufacturer = "Docker" if hw_in_docker else "Unknown"
     hw_product = "Container" if hw_in_docker else "Unknown"
-    hw_uuid = machineid.id()
+    try:
+        hw_uuid = machineid.id()
+    except machineid.MachineIdNotFound:
+        hw_uuid = "Unknown"
 
     if platform.system().lower() == "darwin":
         # Get macOS machine info
@@ -301,7 +304,9 @@ def get_host_stats() -> dict[str, Any]:
 
 
 def get_host_guid() -> str:
-    return machineid.hashed_id("archivebox")
+    from archivebox.config.paths import get_machine_id
+
+    return get_machine_id(length=64)
 
 
 # Example usage

archivebox/machine/migrations/0023_repair_binary_field_indexes.py

@@ -0,0 +1,27 @@
+from django.db import migrations, models
+
+
+def repair_binary_field_indexes(apps, schema_editor):
+    Binary = apps.get_model("machine", "Binary")
+    connection = schema_editor.connection
+    with connection.cursor() as cursor:
+        constraints = connection.introspection.get_constraints(cursor, Binary._meta.db_table)
+    indexed_columns = {
+        tuple(constraint["columns"])
+        for constraint in constraints.values()
+        if constraint.get("index") or constraint.get("unique") or constraint.get("primary_key")
+    }
+    # 0005's legacy raw-SQL table creation omitted field indexes even though
+    # migration state declares them. Editing 0005 alone cannot repair already
+    # upgraded collections. Restore only missing single-field indexes, checking
+    # columns rather than generated names so fresh SQLite/PostgreSQL installs
+    # retain their existing indexes without duplicates or a schema-state change.
+    for field in Binary._meta.local_fields:
+        if field.db_index and not field.unique and (field.column,) not in indexed_columns:
+            schema_editor.add_index(Binary, models.Index(fields=[field.name], name=f"mach_binary_{field.column}_idx"))
+
+
+class Migration(migrations.Migration):
+    dependencies = [("machine", "0022_networkinterface_identity_without_mac")]
+
+    operations = [migrations.RunPython(repair_binary_field_indexes, migrations.RunPython.noop)]

archivebox/machine/models.py

@@ -20,7 +20,7 @@ from django.utils.functional import cached_property
 
 from archivebox.config import CONSTANTS
 from archivebox.config.common import rprint
-from archivebox.base_models.models import ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
+from archivebox.base_models.models import PagedQuerySet, ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
 from archivebox.workers.models import ModelWithQueue
 from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats
 
@@ -48,7 +48,6 @@ _CURRENT_PROCESS: Process | None = None
 MACHINE_RECHECK_INTERVAL = 7 * 24 * 60 * 60
 NETWORK_INTERFACE_RECHECK_INTERVAL = 1 * 60 * 60
 BINARY_RECHECK_INTERVAL = 1 * 30 * 60
-PROCESS_RECHECK_INTERVAL = 60  # Re-validate every 60 seconds
 PID_REUSE_WINDOW = timedelta(hours=24)  # Max age for considering a PID match valid
 PROCESS_TIMEOUT_GRACE = timedelta(seconds=30)  # Extra margin before force-cleaning timed-out RUNNING rows
 START_TIME_TOLERANCE = 5.0  # Seconds tolerance for start time matching
@@ -221,35 +220,21 @@ class Machine(ModelWithHealthStats):
         app_label = "machine"
 
     @classmethod
-    def current(cls, refresh: bool = False) -> Machine:
-        global _CURRENT_MACHINE
-        if refresh:
-            _CURRENT_MACHINE = None
-        if _CURRENT_MACHINE:
-            if timezone.now() < _CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
-                # Reconciliation can save() and clear the shared cache; keep
-                # this call's instance so we never return None after that save.
-                machine = _CURRENT_MACHINE
-                # One-time-per-process reconciliation between ArchiveBox.conf
-                # and Machine.config. Fast-path: bool check + early-return when
-                # the sync has already run, so the cached-machine return path
-                # stays sub-microsecond.
-                try:
-                    from archivebox.config.collection import sync_machine_and_file
+    def current_readonly(cls) -> Machine | None:
+        """Read the current machine without registration, refresh, or config writes."""
+        if _CURRENT_MACHINE is not None:
+            return _CURRENT_MACHINE
+        return cls.objects.filter(guid=get_host_guid()).first()
 
-                    sync_machine_and_file(machine)
-                except Exception:
-                    pass
-                return machine
-            else:
-                _CURRENT_MACHINE = None
+    @classmethod
+    def current(cls) -> Machine:
+        global _CURRENT_MACHINE
+        if _CURRENT_MACHINE is not None:
+            return _CURRENT_MACHINE
 
         host_guid = get_host_guid()
-        # save() intentionally invalidates _CURRENT_MACHINE so config edits
-        # made through other instances become visible in this process. Host
-        # refresh (after seven days), sanitation, and reconciliation can all
-        # save here, so the working instance must not live only in that cache.
-        # Otherwise a routine refresh leaves startup dereferencing None.
+        # Host detection, sanitation, and config reconciliation run only on
+        # initial registration, never on cached reads.
         try:
             machine = cls.objects.get(guid=host_guid)
         except cls.DoesNotExist:
@@ -269,44 +254,41 @@ class Machine(ModelWithHealthStats):
                 **get_vm_info(),
                 stats=get_host_stats(),
             )
-        else:
-            if timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
-                for key, value in {
-                    "hostname": socket.gethostname(),
-                    **get_os_info(),
-                    **get_vm_info(),
-                    "stats": get_host_stats(),
-                }.items():
-                    setattr(machine, key, value)
-                machine.save(
-                    update_fields=[
-                        "hostname",
-                        "hw_in_docker",
-                        "hw_in_vm",
-                        "hw_manufacturer",
-                        "hw_product",
-                        "hw_uuid",
-                        "os_arch",
-                        "os_family",
-                        "os_platform",
-                        "os_release",
-                        "os_kernel",
-                        "stats",
-                        "modified_at",
-                    ],
-                )
-        machine = cls._sanitize_config(machine)
-        # Same one-time sync as the cached-return path. Triggers here on the
-        # very first ``Machine.current()`` call in a process before any
-        # cached return can occur.
+        needs_recheck = timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL)
+        # Reconcile before any save can mirror stale DB config over file edits.
         try:
             from archivebox.config.collection import sync_machine_and_file
 
             sync_machine_and_file(machine)
         except Exception:
             pass
-        # Publish only after all save-capable steps; publishing earlier would
-        # let those steps invalidate the cache we are in the middle of filling.
+        if needs_recheck:
+            for key, value in {
+                "hostname": socket.gethostname(),
+                **get_os_info(),
+                **get_vm_info(),
+                "stats": get_host_stats(),
+            }.items():
+                setattr(machine, key, value)
+            machine.save(
+                update_fields=[
+                    "hostname",
+                    "hw_in_docker",
+                    "hw_in_vm",
+                    "hw_manufacturer",
+                    "hw_product",
+                    "hw_uuid",
+                    "os_arch",
+                    "os_family",
+                    "os_platform",
+                    "os_release",
+                    "os_kernel",
+                    "stats",
+                    "modified_at",
+                ],
+            )
+        machine = cls._sanitize_config(machine)
+        # Publish only after initialization is complete.
         _CURRENT_MACHINE = machine
         return machine
 
@@ -377,18 +359,15 @@ class Machine(ModelWithHealthStats):
             if update_fields is not None:
                 kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
 
-        # Drop the ``Machine.current()`` module-level cache on every save so
-        # config edits (admin form, from_json, etc.) become live in the same
-        # process without waiting out the 7-day ``MACHINE_RECHECK_INTERVAL``.
-        # The cache reads ``_CURRENT_MACHINE.modified_at`` and that value
-        # never moves forward on the cached object even when the row is
-        # updated in the DB, so without this we'd keep serving stale
-        # ``machine.config`` (incl. ``BASE_URL``) until the worker restarts.
+        # Apply explicit edits to the cached instance without making the next
+        # read repeat host detection, sanitation, or config reconciliation.
         update_fields = kwargs.get("update_fields")
         super().save(*args, **kwargs)
         global _CURRENT_MACHINE
         if _CURRENT_MACHINE is not None and _CURRENT_MACHINE.pk == self.pk:
-            _CURRENT_MACHINE = None
+            for field in self._meta.concrete_fields:
+                if field.attname in self.__dict__ and (update_fields is None or field.name in update_fields):
+                    setattr(_CURRENT_MACHINE, field.attname, self.__dict__[field.attname])
 
         # Mirror Machine.config into ArchiveBox.conf so the two stores stay
         # 1:1. Skipped when ``update_fields`` is set and doesn't touch
@@ -443,7 +422,7 @@ class NetworkInterface(ModelWithHealthStats):
     @classmethod
     def current(cls, refresh: bool = False) -> NetworkInterface:
         global _CURRENT_INTERFACE
-        machine = Machine.current(refresh=refresh)
+        machine = Machine.current()
         if _CURRENT_INTERFACE and _CURRENT_INTERFACE.machine_id == machine.id:
             if not refresh:
                 # Callers that pass refresh=False are asking for attribution to
@@ -493,7 +472,7 @@ class NetworkInterface(ModelWithHealthStats):
         return _CURRENT_INTERFACE
 
 
-class BinaryManager(models.Manager):
+class BinaryManager(models.Manager.from_queryset(PagedQuerySet)):
     def get_from_db_or_cache(self, name: str, abspath: str = "", version: str = "", sha256: str = "", binprovider: str = "env") -> Binary:
         """Get or create an Binary record from the database or cache."""
         cached = _CURRENT_BINARIES.get(name)
@@ -910,7 +889,7 @@ class Binary(ModelWithHealthStats, ModelWithQueue):
 # =============================================================================
 
 
-class ProcessManager(models.Manager):
+class ProcessManager(models.Manager.from_queryset(PagedQuerySet)):
     """Manager for Process model."""
 
     def current(self) -> Process:
@@ -1229,8 +1208,11 @@ class Process(ModelWithDeleteAfter, models.Model):
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(env__has_key="DELETE_AFTER") | Q(machine__config__has_key="DELETE_AFTER"),
+        return (
+            cls.objects.filter(delete_at__isnull=True)
+            .filter(Q(env__has_key="DELETE_AFTER") | Q(machine__config__has_key="DELETE_AFTER"))
+            .select_related("machine")
+            .only("id", "created_at", "delete_at", "env", "machine__config")
         )
 
     # Properties that delegate to related objects
@@ -1430,6 +1412,13 @@ class Process(ModelWithDeleteAfter, models.Model):
     # Process.current() and hierarchy methods
     # =========================================================================
 
+    @classmethod
+    def current_readonly(cls) -> Process | None:
+        """Return this process's registered identity without registration or probes."""
+        if _CURRENT_PROCESS is not None and _CURRENT_PROCESS.pid == os.getpid():
+            return _CURRENT_PROCESS
+        return None
+
     @classmethod
     def current(cls) -> Process:
         """
@@ -1447,40 +1436,19 @@ class Process(ModelWithDeleteAfter, models.Model):
 
         current_pid = os.getpid()
 
-        # Fast path used by model save diagnostics and hot runner loops. A
-        # cached Process object is valid when the immutable identity we wrote
-        # at creation time still describes this Python process. PID reuse cannot
-        # happen while this process is alive, so pid + present started_at/cmd is
-        # enough here; the slower psutil validation below remains the fallback
-        # for missing/stale cache.
-        if (
-            _CURRENT_PROCESS
-            and _CURRENT_PROCESS.pid == current_pid
-            and _CURRENT_PROCESS.status == cls.StatusChoices.RUNNING
-            and timezone.now() < _CURRENT_PROCESS.modified_at + timedelta(seconds=PROCESS_RECHECK_INTERVAL)
-            and _CURRENT_PROCESS.started_at is not None
-            and bool(_CURRENT_PROCESS.cmd)
-        ):
-            return _CURRENT_PROCESS
+        # This is process identity, not a health check. A live Python process
+        # cannot reuse its own PID, and a fork fails current_readonly's PID
+        # check. Expiring this cache by modified_at made every call after 60s
+        # query the database and touch logs until something else saved the row.
+        # Explicit lifecycle/recovery code owns liveness and interface refresh.
+        cached = cls.current_readonly()
+        if cached is not None:
+            return cached
+        _CURRENT_PROCESS = None
 
         machine = Machine.current()
         iface = NetworkInterface.current()
 
-        # Check cache validity
-        if _CURRENT_PROCESS:
-            # Verify: same PID, same machine, cache not expired
-            if (
-                _CURRENT_PROCESS.pid == current_pid
-                and _CURRENT_PROCESS.machine_id == machine.id
-                and timezone.now() < _CURRENT_PROCESS.modified_at + timedelta(seconds=PROCESS_RECHECK_INTERVAL)
-            ):
-                if _CURRENT_PROCESS.iface_id != iface.id:
-                    _CURRENT_PROCESS.iface = iface
-                    _CURRENT_PROCESS.save(update_fields=["iface", "modified_at"])
-                _CURRENT_PROCESS.ensure_log_files()
-                return _CURRENT_PROCESS
-            _CURRENT_PROCESS = None
-
         # Get actual process start time from OS for validation
         os_start_time = None
         if PSUTIL_AVAILABLE:
@@ -1660,9 +1628,8 @@ class Process(ModelWithDeleteAfter, models.Model):
         if machine is not None:
             stale = stale.filter(machine=machine)
 
-        # Recovery can run against damaged DB state; stream rows so a large
-        # stale Process backlog cannot be materialized in memory at once.
-        for proc in stale.iterator(chunk_size=100):
+        # Close each bounded read before inspecting processes or writing their state.
+        for proc in stale.order_by("pk").paged_iterator(chunk_size=100):
             shares_pid_namespace = proc.shares_pid_namespace
             if shares_pid_namespace and proc.poll() is not None:
                 cleaned += 1
@@ -1778,7 +1745,8 @@ class Process(ModelWithDeleteAfter, models.Model):
             return None
 
         # Can't validate processes on other machines
-        if self.machine_id != Machine.current().id:
+        machine = Machine.current_readonly()
+        if machine is None or self.machine_id != machine.id:
             return None
 
         try:
@@ -2601,14 +2569,17 @@ class Process(ModelWithDeleteAfter, models.Model):
         """
         cleaned = 0
 
+        # Select active IDs through the existing status index first. Combining
+        # type/status directly let SQLite prefer the historical type index and
+        # scan 350k processes to return zero workers (blocking writes for 12s).
+        active_ids = cls.objects.filter(status=cls.StatusChoices.RUNNING).order_by().values("pk")
         running_children = cls.objects.filter(
+            pk__in=active_ids,
             process_type__in=[cls.TypeChoices.WORKER, cls.TypeChoices.HOOK],
-            status=cls.StatusChoices.RUNNING,
         )
 
-        # Recovery can run against damaged DB state; stream rows so a large
-        # orphaned Process backlog cannot be materialized in memory at once.
-        for proc in running_children.iterator(chunk_size=100):
+        # Close each bounded read before inspecting processes or writing their state.
+        for proc in running_children.order_by("pk").paged_iterator(chunk_size=100):
             if not proc.is_running:
                 proc.mark_exited(
                     exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),

archivebox/misc/checks.py

@@ -194,7 +194,10 @@ def check_not_root():
     if os.geteuid() == 0 and not (is_getting_help or is_getting_version):
         print("[yellow][!] Running ArchiveBox as root is not recommended.[/yellow]", file=sys.stderr)
         print("    Root-owned DATA_DIR files may be inaccessible to non-root users later.", file=sys.stderr)
-        print("        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root", file=sys.stderr)
+        print(
+            "        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user",
+            file=sys.stderr,
+        )
 
 
 def is_archivebox_source_root(path: Path | str | None = None) -> bool:
@@ -255,7 +258,7 @@ def check_data_dir_permissions(config=None, **config_kwargs):
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions]https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions[/link]",
         )
         STDERR.print(
-            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid[/link]",
+            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues[/link]",
         )
         STDERR.print(
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts[/link]",

archivebox/misc/db.py

@@ -240,89 +240,38 @@ def drop_models_on_postgres(apps, schema_editor, app_label: str, model_names: li
             schema_editor.delete_model(model)
 
 
-def run_db_analyze_batch(
-    remaining: list[str] | None,
-    *,
-    max_seconds_per_table: float = 120.0,
-) -> list[str]:
-    """Advance one step of a batched SQLite ``ANALYZE`` sweep.
+def run_db_analyze_batch(remaining: list[str] | None) -> list[str]:
+    """Refresh approximate SQLite planner stats for one table per idle tick.
 
-    Without periodic ANALYZE the optimizer's table stats go stale as
-    snapshot/archiveresult tables grow, causing it to start large joins from
-    ``auth_user`` instead of using the indexed url column and blowing snapshot
-    detail page render time from ~50ms to ~500ms+.
-
-    The whole sweep is spread across many calls instead of running as one
-    blocking ``ANALYZE``: pass ``None`` to start a fresh sweep (this call
-    enumerates user tables and runs ``ANALYZE`` on the first one); pass the
-    returned list to advance one more table on each subsequent call. An
-    empty return value means the sweep is complete (or has been aborted) and
-    the next caller should pass ``None`` again. Caller is responsible for
-    throttling new sweeps (orchestrator starts at most one per 24hr while
-    idle) and enforcing a hard upper bound on total sweep wall time.
-
-    Safety guarantees:
-
-    - **Never raises**: every database call is wrapped; on any failure the
-      function returns ``[]`` (abandoning the rest of the sweep) so the
-      orchestrator never crashes on maintenance errors.
-    - **Bounded per-call wall time**: a SQLite progress handler aborts the
-      current ``ANALYZE`` statement once ``max_seconds_per_table`` is
-      exceeded, so a single pathological table cannot wedge the call.
-    - **Never leaves the db locked**: each ``ANALYZE`` runs as a single
-      statement transaction that auto-commits (or rolls back on
-      abort/error). The cursor and progress handler are always cleaned up
-      in ``finally`` blocks even if Python raises mid-call.
-    - Silent no-op on non-SQLite backends.
-
-    WAL journal mode (set in Django settings) keeps readers fully unblocked
-    throughout; the writer lock is only held for the brief ``sqlite_stat*``
-    flush after each table completes.
+    A full ANALYZE scans every index while holding a write transaction; moving
+    it into the runner did not prevent it from blocking submissions. Sampling
+    at most about 1000 entries per index keeps maintenance independent of
+    collection size. Approximate stats are sufficient for choosing indexes.
+    PostgreSQL owns its statistics through autovacuum/autoanalyze.
     """
     from django.db import connection
 
     if connection.vendor != "sqlite":
         return []
-
-    if remaining is None:
-        try:
-            with connection.cursor() as cursor:
-                cursor.execute(
-                    "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
-                )
-                remaining = [row[0] for row in cursor.fetchall()]
-        except Exception:
-            return []
-
-    if not remaining:
-        return []
-
-    next_table, *rest = remaining
-    raw_conn = connection.connection
-    progress_handler_set = False
-    if raw_conn is not None and max_seconds_per_table > 0:
-        deadline = time.monotonic() + max_seconds_per_table
-        try:
-            raw_conn.set_progress_handler(lambda: 1 if time.monotonic() > deadline else 0, 10000)
-            progress_handler_set = True
-        except Exception:
-            progress_handler_set = False
-
     try:
         with connection.cursor() as cursor:
-            cursor.execute(f'ANALYZE "{next_table}"')
-    except Exception:
-        # Aborted by progress handler, locked db, or any other failure — skip
-        # this table and continue the sweep. ANALYZE is idempotent so we can
-        # retry on the next 24hr sweep.
-        pass
-    finally:
-        if progress_handler_set and raw_conn is not None:
+            if remaining is None:
+                cursor.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")
+                remaining = [row[0] for row in cursor.fetchall()]
+            if not remaining:
+                return []
+            next_table, *rest = remaining
+            cursor.execute("PRAGMA analysis_limit")
+            previous_limit = int(cursor.fetchone()[0])
             try:
-                raw_conn.set_progress_handler(None, 0)
-            except Exception:
-                pass
-    return rest
+                cursor.execute("PRAGMA analysis_limit = 1000")
+                cursor.execute(f"ANALYZE {connection.ops.quote_name(next_table)}")
+            finally:
+                cursor.execute(f"PRAGMA analysis_limit = {previous_limit}")
+            return rest
+    except Exception:
+        # Optional planner maintenance must not prevent queued captures.
+        return []
 
 
 def compact_command(cmdline: list[str] | None, fallback: str = "") -> str:
@@ -445,7 +394,10 @@ def migration_lock(stdout: TextIO | None = None):
 
 
 # Migration names that previously existed in ArchiveBox's source tree but have
-# since been deleted (squashed away, renamed, moved between apps, etc.). DBs
+# since been deleted (squashed away, renamed, moved between apps, etc.), plus
+# exact older migrations generated in the installed package by v0.6.2's
+# startup makemigrations. Those runtime-generated names were never in Git, so
+# the deleted-file history alone cannot cover them. DBs
 # upgraded incrementally through 0.8.x → 0.9.x dev rcs accumulate rows for
 # these in ``django_migrations``; the newer-DB guard added in 65dc2521 would
 # otherwise refuse to start with "applied migrations missing from this build"
@@ -454,13 +406,18 @@ def migration_lock(stdout: TextIO | None = None):
 # splits the migration graph when only a *subset* of the replaces list is
 # applied (which is exactly what happens for users at different intermediate
 # dev branch states). This set is the authoritative compat list — extend it
-# when squashing more migrations away. Generated from
-# ``git log --diff-filter=D --name-only`` over each app's migrations/ tree.
+# when squashing more migrations away. The source-tracked names were generated
+# from ``git log --diff-filter=D --name-only`` over each app's migrations/ tree.
 HISTORICAL_GHOST_MIGRATIONS: frozenset[tuple[str, str]] = frozenset(
     {
         # core: 0023→0075 sequence plus every transient dev rename
         ("core", "0002_auto_20190417_0739"),
         ("core", "0006_auto_20200915_2006"),
+        # Reported v0.6.2 runtime-generated 0021 migrations; the checked-in
+        # replacement only changes ArchiveResult.extractor choices.
+        ("core", "0021_auto_20220510_0644"),
+        ("core", "0021_auto_20220724_1254"),
+        ("core", "0021_auto_20221128_1116"),
         ("core", "0023_alter_archiveresult_options_archiveresult_abid_and_more"),
         ("core", "0023_new_schema"),
         ("core", "0024_auto_20240513_1143"),
@@ -518,6 +475,8 @@ HISTORICAL_GHOST_MIGRATIONS: frozenset[tuple[str, str]] = frozenset(
         ("core", "0053_remove_snapshottag_snapshot_old"),
         ("core", "0054_alter_snapshot_timestamp"),
         ("core", "0055_alter_tag_slug"),
+        # Dev-only progress indexes removed in favor of existing-index queries.
+        ("core", "0056_progress_covering_indexes"),
         ("core", "0056_remove_tag_uuid"),
         ("core", "0057_rename_id_tag_old_id"),
         ("core", "0058_alter_tag_old_id"),

archivebox/misc/jsonl.py

@@ -142,3 +142,27 @@ def write_record(record: dict[str, Any], stream: TextIO | None = None) -> None:
     active_stream: TextIO = sys.stdout if stream is None else stream
     active_stream.write(json.dumps(record) + "\n")
     active_stream.flush()
+
+
+def write_records(records: Iterable[dict[str, Any]], stream: TextIO | None = None) -> int:
+    """Stream bulk JSONL output with bounded buffering, returning its row count."""
+    active_stream: TextIO = sys.stdout if stream is None else stream
+    lines: list[str] = []
+    pending_size = 0
+    count = 0
+    for record in records:
+        line = json.dumps(record) + "\n"
+        lines.append(line)
+        pending_size += len(line)
+        count += 1
+        # CLI stdout can be unbuffered. A syscall per snapshot makes large
+        # exports IO-bound; small batches keep memory and pipe latency bounded.
+        if pending_size >= 64 * 1024:
+            active_stream.write("".join(lines))
+            active_stream.flush()
+            lines.clear()
+            pending_size = 0
+    if lines:
+        active_stream.write("".join(lines))
+        active_stream.flush()
+    return count

archivebox/misc/serve_static.py

@@ -18,7 +18,6 @@ from html.parser import HTMLParser
 from pathlib import Path
 from urllib.parse import quote, urlencode, urljoin
 
-from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay
 from django.contrib.staticfiles import finders
 from django.core.handlers.asgi import ASGIRequest
 from django.http import Http404, HttpResponse, HttpResponseNotModified, StreamingHttpResponse
@@ -30,8 +29,11 @@ from django.views import static
 
 from archivebox.config.common import get_config
 from archivebox.misc.logging_util import printable_filesize
+from archivebox.plugins.discovery import render_plugin_full_response
 
 _HASHES_CACHE: dict[Path, tuple[float, dict[str, str]]] = {}
+DIRECTORY_PREVIEW_MAX_BYTES = 100 * 1024
+FAVICON_CACHE_CONTROL = "public, max-age=31536000, s-maxage=31536000, immutable"
 IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\s(?:src|data-src)=["\'])([^"\']+)(["\'])', re.IGNORECASE)
 TRANSFORMED_HTML_PREVIEW_STYLE = """<style id="archivebox-static-html-preview-style">
 html {
@@ -68,6 +70,192 @@ a > img:not([width]):not([height]) {
 </style>"""
 
 
+def add_replay_image_fallback(response: HttpResponse) -> HttpResponse:
+    """Add a best-effort responsive-image fallback to the locally served replay worker.
+
+    WHY: a recorder saves the image Chrome selected, not every srcset candidate.
+    Replaying at another viewport/DPR can select an uncaptured URL (the motivating
+    case was Drive's missing 2x logo with its 1x logo already in the same WACZ).
+    WACZ requests are answered inside Webrecorder's service worker, so a Django
+    404 handler cannot repair them. Append this adapter when serving that worker;
+    do not change captures, request more live assets, or resize the recorded page.
+
+    Only an image GET that already returned 404 is eligible. The worker reads the
+    referring HTML from the SAME archive and tries src/srcset candidates declared
+    on that img or its enclosing picture. It never guesses filename substitutions
+    or borrows an unrelated nearby image. The original response wins on absent
+    metadata, unsupported worker APIs, parse errors, timeout, or failed candidates.
+    This deliberately does not promise to fix JS-only srcsets, CSS backgrounds,
+    corrupt images returning 200, or pages whose archived HTML is too large.
+
+    Keep this self-contained in the static-file server: it is a replay repair,
+    independent of the capturing plugin/site. Feature detection makes upstream
+    worker changes disable the repair rather than break ordinary replay. The
+    archive's HTML/ZIP bytes and successful HTTP responses remain untouched.
+    """
+    if (
+        response.status_code != 200
+        or response.streaming
+        or not response.has_header("Service-Worker-Allowed")
+        or "javascript" not in response.get("Content-Type", "")
+    ):
+        return response
+    script = rb"""
+;(() => {
+  try {
+    const replay = self.sw;
+    if (!replay || typeof replay.handleFetch !== "function" ||
+        typeof replay.getResponseFor !== "function" || !replay.collections ||
+        replay.proxyOriginMode || replay.topFramePassthrough || replay.__abxImageFallback) return;
+    const original = replay.handleFetch;
+    const documents = new Map();
+    const decode = value => value.replace(/&(?:amp|quot|apos|lt|gt|#\d+|#x[\da-f]+);/gi, entity => {
+      const named = {"&amp;":"&", "&quot;":'"', "&apos;":"'", "&lt;":"<", "&gt;":">"};
+      return named[entity.toLowerCase()] || String.fromCodePoint(
+        entity[2].toLowerCase() === "x" ? parseInt(entity.slice(3), 16) : parseInt(entity.slice(2), 10));
+    });
+    const attributes = tag => {
+      const attrs = {};
+      for (const m of tag.matchAll(/([^\s=/>]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+))/g)) {
+        attrs[m[1].toLowerCase()] = decode(m[2] ?? m[3] ?? m[4]);
+      }
+      return attrs;
+    };
+    const sourceSet = value => {
+      // URLs may contain commas (e.g. image-CDN transforms). Like HTML's
+      // srcset tokenizer, consume a URL through whitespace, then descriptors.
+      const urls = [];
+      let rest = value || "";
+      while (rest && urls.length < 64) {
+        rest = rest.replace(/^[\s,]+/, "");
+        const token = rest.match(/^\S+/)?.[0];
+        if (!token) break;
+        urls.push(token.replace(/,+$/, ""));
+        rest = rest.slice(token.length);
+        if (!token.endsWith(",")) rest = rest.replace(/^[^,]*(?:,|$)/, "");
+      }
+      return urls;
+    };
+    const splitReplay = (url, prefix) => {
+      if (!url.startsWith(prefix)) return null;
+      return url.slice(prefix.length).match(/^((?::[^/]+\/)?\d{0,14})(?:[a-z]+_)?\/((?:https?:)?\/\/.+)$/);
+    };
+    replay.handleFetch = async function(event) {
+      // Never catch/retry the original operation: preserve its normal behavior.
+      const response = await original.call(this, event);
+      const request = event.request;
+      if (response.status !== 404 || request.method !== "GET" || request.destination !== "image") return response;
+      let expired = false, timer;
+      try {
+        const repair = async () => {
+          if (!request.url.startsWith(this.replayPrefix)) return response;
+          const name = this.collections.root || request.url.slice(this.replayPrefix.length).split("/", 1)[0];
+          const collection = await this.collections.getColl(name);
+          // Do not operate on live/proxy collections or remote source archives.
+          // lookupUrl is the archive index API, absent on the live proxy store.
+          if (!collection || typeof collection.store?.lookupUrl !== "function" || collection.liveRedirectOnNotFound ||
+              new URL(collection.config.sourceUrl).origin !== self.location.origin) return response;
+          const missing = splitReplay(request.url, collection.prefix);
+          const referring = splitReplay(request.referrer, collection.prefix);
+          if (!missing || !referring) return response;
+          const archiveURL = url => collection.prefix + referring[1] + "id_/" + url;
+          const read = url => this.getResponseFor(new Request(archiveURL(url), {
+            credentials: "same-origin", redirect: "manual", signal: request.signal,
+          }), event);
+          const key = archiveURL(referring[2]);
+          if (!documents.has(key)) {
+            // Cache only candidate metadata, never whole HTML or image bodies.
+            // Both entry count and HTML reads are bounded; WACZ range loading
+            // remains the upstream replay engine's responsibility.
+            if (documents.size >= 16) documents.delete(documents.keys().next().value);
+            documents.set(key, (async () => {
+              const html = await read(referring[2]);
+              if (html.status !== 200 || !html.headers.get("content-type")?.includes("text/html")) return [];
+              const reader = html.body.getReader(), decoder = new TextDecoder();
+              let text = "", size = 0;
+              try {
+                while (true) {
+                  const chunk = await reader.read();
+                  if (chunk.done) break;
+                  size += chunk.value.byteLength;
+                  if (expired || size > 2 * 1024 * 1024) return [];
+                  text += decoder.decode(chunk.value, {stream:true});
+                }
+                text += decoder.decode();
+              } finally {await reader.cancel().catch(() => {});}
+              // Ignore tag-shaped strings in scripts/comments. Parsing is
+              // intentionally conservative; an odd/malformed page stays broken
+              // instead of acquiring an unrelated "similar looking" image.
+              text = text.replace(/<!--[\s\S]*?-->|<(script|style)\b[^>]*>[\s\S]*?<\/\1\s*>/gi, "");
+              const groups = [];
+              let base = referring[2], picture = null;
+              const absolute = value => {
+                try {
+                  const url = new URL(value, base);
+                  if (!/^https?:$/.test(url.protocol) || url.username || url.password) return null;
+                  url.hash = "";
+                  return url.href;
+                } catch {return null;}
+              };
+              for (const token of text.matchAll(/<\/?(?:base|picture|source|img)\b(?:"[^"]*"|'[^']*'|[^'">])*>/gi)) {
+                const tag = token[0], attrs = attributes(tag);
+                if (/^<base\b/i.test(tag)) {if (attrs.href) base = absolute(attrs.href) || base; continue;}
+                if (/^<picture\b/i.test(tag)) {picture = []; continue;}
+                if (/^<\/picture\b/i.test(tag)) {picture = null; continue;}
+                const urls = [...new Set([attrs.src, ...sourceSet(attrs.srcset)].filter(Boolean).map(absolute).filter(Boolean))];
+                if (/^<source\b/i.test(tag)) {if (picture) picture.push(...urls); continue;}
+                if (!attrs.srcset && !picture?.length) continue;
+                groups.push([...new Set([...urls, ...(picture || [])])].slice(0, 64));
+                if (groups.length >= 512) break;
+              }
+              return groups;
+            })());
+          }
+          const groups = await documents.get(key);
+          const failed = new URL(missing[2], referring[2]); failed.hash = "";
+          const candidates = groups.find(group => group.includes(failed.href));
+          if (!candidates) return response;
+          // Prefer the img's explicit src, then alternatives in declaration
+          // order. Limit extra archive lookups, and never follow redirects.
+          for (const url of candidates.filter(url => url !== failed.href).slice(0, 6)) {
+            if (expired) break;
+            const alternative = await read(url);
+            if (alternative.status !== 200 || !alternative.headers.get("content-type")?.startsWith("image/")) continue;
+            if (expired) break;
+            const headers = new Headers(alternative.headers);
+            headers.set("X-ArchiveBox-Image-Fallback", url);
+            headers.set("Cache-Control", "no-store");
+            return new Response(alternative.body, {status:200, headers});
+          }
+          return response;
+        };
+        return await Promise.race([
+          repair().catch(() => response),
+          new Promise(resolve => {timer = setTimeout(() => {expired = true; resolve(response);}, 1500);}),
+        ]);
+      } catch {return response;}
+      finally {expired = true; clearTimeout(timer);}
+    };
+    replay.__abxImageFallback = true;
+  } catch { /* Optional repair must never prevent the worker from starting. */ }
+})();
+"""
+    try:
+        body = response.content
+        if b"self.sw=" not in body or b"__abxImageFallback" in body:
+            return response
+        response.content = body + script
+        # Upstream headers describe the unmodified worker. Revalidate this
+        # generated asset so existing captures can receive the repair too.
+        for header in ("ETag", "Last-Modified", "Content-Length"):
+            if response.has_header(header):
+                del response[header]
+        response["Cache-Control"] = "no-cache"
+    except Exception:
+        pass
+    return response
+
+
 def _load_hash_map(snapshot_dir: Path) -> dict[str, str] | None:
     hashes_path = snapshot_dir / "hashes" / "hashes.json"
     if not hashes_path.exists():
@@ -125,7 +313,8 @@ def _resolve_archive_path(document_root: str | Path, rel_path: str) -> tuple[Pat
         current = match
         resolved_parts.append(match.name)
 
-    return current, posixpath.join(*resolved_parts) if resolved_parts else ""
+    resolved_path = posixpath.join(*resolved_parts) if resolved_parts else ""
+    return Path(safe_join(document_root, resolved_path)), resolved_path
 
 
 def _cache_policy(config=None, **config_kwargs) -> str:
@@ -307,10 +496,24 @@ def _render_directory_index(request, path: str, fullpath: Path) -> HttpResponse:
         file_list.append(url)
 
         stat_result = entry.stat()
+        preview_kind = ""
+        mime = mimetypes.guess_type(entry.name)[0] or "application/octet-stream"
+        if entry.is_file() and stat_result.st_size < DIRECTORY_PREVIEW_MAX_BYTES:
+            if mime.startswith("image/"):
+                preview_kind = "image"
+            elif (
+                mime.startswith("text/")
+                or re.search(r"(?:^|[.+/-])(?:json|xml|javascript|ecmascript|yaml|toml|graphql|sql)(?:[.+/-]|$)", mime)
+                or entry.suffix.lower() in {".log", ".jsonl", ".toml"}
+            ):
+                preview_kind = "text"
         entries.append(
             {
                 "name": url,
                 "url": url,
+                "preview_kind": preview_kind,
+                "mimetype": "Directory" if entry.is_dir() else mime,
+                "size_bytes": 0 if entry.is_dir() else stat_result.st_size,
                 "is_dir": entry.is_dir(),
                 "size": "—" if entry.is_dir() else printable_filesize(stat_result.st_size),
                 "timestamp": _format_direntry_timestamp(stat_result),
@@ -850,7 +1053,8 @@ def _is_risky_replay_document(fullpath: Path, content_type: str) -> bool:
     # Unknown archived response paths often have no extension. Sniff a small prefix
     # so one-domain no-JS mode still catches HTML/SVG documents.
     try:
-        head = fullpath.read_bytes()[:4096].decode("utf-8", errors="ignore").lower()
+        with fullpath.open("rb") as source:
+            head = source.read(4096).decode("utf-8", errors="ignore").lower()
     except (OSError, UnicodeDecodeError):
         return False
 
@@ -939,6 +1143,15 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
         raise Http404(_("“%(path)s” does not exist") % {"path": fullpath})
 
     statobj = fullpath.stat()
+    # Captured site icons are intentionally shareable even for private snapshots.
+    # Only raw icon responses qualify, not generated previews or directory pages.
+    favicon_cache_control = (
+        FAVICON_CACHE_CONTROL
+        if fullpath.stem.lower() == "favicon"
+        and fullpath.suffix.lower() in {".ico", ".svg", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif"}
+        and not request.GET.get("preview")
+        else None
+    )
     document_root = Path(document_root) if document_root else None
     rel_path = path
     etag = None
@@ -954,7 +1167,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             if etag in inm_list or etag.strip('"') in [i.strip('"') for i in inm_list]:
                 not_modified = HttpResponseNotModified()
                 not_modified.headers["ETag"] = etag
-                not_modified.headers["Cache-Control"] = f"{cache_policy}, max-age=31536000, immutable"
+                not_modified.headers["Cache-Control"] = favicon_cache_control or f"{cache_policy}, max-age=31536000, immutable"
                 not_modified.headers["Last-Modified"] = http_date(statobj.st_mtime)
                 return _apply_archive_replay_headers(
                     not_modified,
@@ -965,6 +1178,13 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 )
 
     content_type, encoding = mimetypes.guess_type(str(fullpath))
+    # Captured favicons retain the conventional .ico filename even when a site
+    # returns SVG. Browsers cannot decode SVG served as image/x-icon.
+    if fullpath.suffix.lower() == ".ico":
+        with fullpath.open("rb") as icon_file:
+            icon_head = icon_file.read(4096)
+        if re.search(rb"<svg(?:\s|>)", icon_head):
+            content_type = "image/svg+xml"
     preserve_plain_text = fullpath.suffix.lower() in {".log", ".sh", ".jsonl"}
     if preserve_plain_text:
         content_type = "text/plain"
@@ -988,15 +1208,13 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
     preview_as_image_html = (
         bool(request.GET.get("preview")) and content_type.startswith("image/") and not content_type.startswith("image/svg+xml")
     )
-    preview_as_archivewebpage_html = bool(request.GET.get("preview")) and archivewebpage_replay.is_replay_target(fullpath.name)
-
     # Respect the If-Modified-Since header for non-markdown responses.
     if not content_type.startswith(("text/plain", "text/html")) and not static.was_modified_since(
         request.META.get("HTTP_IF_MODIFIED_SINCE"),
         statobj.st_mtime,
     ):
         not_modified = HttpResponseNotModified()
-        not_modified.headers["Cache-Control"] = f"{cache_policy}, max-age=60, stale-while-revalidate=300"
+        not_modified.headers["Cache-Control"] = favicon_cache_control or f"{cache_policy}, max-age=60, stale-while-revalidate=300"
         return _apply_archive_replay_headers(
             not_modified,
             fullpath=fullpath,
@@ -1005,6 +1223,29 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             config=config,
         )
 
+    # A shared archive explorer for ZIP outputs from any plugin. Raw/range
+    # requests retain the original bytes for downloads and seekable reads.
+    if fullpath.suffix.lower() == ".zip" and request.GET.get("preview") and not request.META.get("HTTP_RANGE"):
+        raw_query = request.GET.copy()
+        raw_query.pop("preview", None)
+        raw_query["raw"] = "1"
+        wrapped = loader.get_template("static/zip_index.html").render(
+            {
+                "title": fullpath.name,
+                "output_path": f"{request.path}?{raw_query.urlencode()}",
+                "archive_size": statobj.st_size,
+            },
+        )
+        response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
+        _set_transformed_response_headers(response, fullpath, statobj, None, cache_policy)
+        return _apply_archive_replay_headers(
+            response,
+            fullpath=fullpath,
+            content_type="text/html; charset=utf-8",
+            is_archive_replay=is_archive_replay,
+            config=config,
+        )
+
     # Wrap text-like outputs in HTML when explicitly requested for iframe previewing.
     if preview_as_text_html:
         try:
@@ -1022,7 +1263,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                     config=config,
                 )
         except (OSError, UnicodeDecodeError, ValueError):
-            preview_as_text_html = False
+            pass
 
     if preview_as_image_html:
         try:
@@ -1042,16 +1283,16 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 config=config,
             )
         except (OSError, ValueError):
-            preview_as_image_html = False
+            pass
 
-    if preview_as_archivewebpage_html:
+    if request.GET.get("preview"):
         try:
             raw_query = request.GET.copy()
             raw_query.pop("preview", None)
             raw_output_path = request.path
             if raw_query:
                 raw_output_path = f"{raw_output_path}?{raw_query.urlencode()}"
-            body, preview_content_type, headers = archivewebpage_replay.render_preview_response(
+            plugin_replay = render_plugin_full_response(
                 fullpath.name,
                 raw_output_path,
                 wacz_path=fullpath,
@@ -1063,18 +1304,20 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 ),
                 content_encoding=encoding or "",
             )
-            response = HttpResponse(body, content_type=preview_content_type)
-            for key, value in headers.items():
-                response.headers[key] = value
-            return _apply_archive_replay_headers(
-                response,
-                fullpath=fullpath,
-                content_type=preview_content_type,
-                is_archive_replay=is_archive_replay,
-                config=config,
-            )
+            if plugin_replay is not None:
+                body, replay_content_type, headers = plugin_replay
+                response = HttpResponse(body, content_type=replay_content_type)
+                for key, value in headers.items():
+                    response.headers[key] = value
+                return _apply_archive_replay_headers(
+                    response,
+                    fullpath=fullpath,
+                    content_type=replay_content_type,
+                    is_archive_replay=is_archive_replay,
+                    config=config,
+                )
         except (OSError, RuntimeError, ValueError):
-            preview_as_archivewebpage_html = False
+            pass
 
     # Heuristic fix: some archived HTML outputs are stored with HTML-escaped markup
     # or markdown sources. If so, render sensibly.
@@ -1146,6 +1389,8 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
         response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"'
     if content_type.startswith("image/"):
         response.headers["Cache-Control"] = f"{cache_policy}, max-age=604800, immutable"
+    if favicon_cache_control:
+        response.headers["Cache-Control"] = favicon_cache_control
 
     # handle byte-range requests by serving chunk of file
     if stat.S_ISREG(statobj.st_mode):

archivebox/misc/shell_welcome_message.py

@@ -48,10 +48,10 @@ if __name__ == "__main__":
 
     prnt("[i] :heavy_dollar_sign: Welcome to the ArchiveBox Shell!")
     prnt(
-        "    [deep_sky_blue4]Docs:[/deep_sky_blue4] [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage[/link]",
+        "    [deep_sky_blue4]Docs:[/deep_sky_blue4] [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage[/link]",
     )
     prnt(
-        "          [link=https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html]https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html[/link]",
+        "          [link=https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html]https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html[/link]",
     )
     prnt()
     prnt(" :grey_question: [violet]Hint[/] [i]Here are some examples to get started:[/]")

archivebox/misc/util.py

@@ -3,7 +3,7 @@ __package__ = "archivebox.misc"
 # Bootable utility functions (URL parsing, date parsing, JSON encoding, decorators).
 # MUST NOT import archivebox.config, archivebox.core, or Django — this module is
 # loaded by hooks.py and other early code paths. Only depends on stdlib and the
-# bootable .logging module.
+# bootable .logging module and standalone abx_plugins URL helpers.
 
 import re
 import json as pyjson
@@ -19,6 +19,9 @@ from urllib.parse import urlparse, quote, unquote
 from html import escape, unescape
 from datetime import datetime, timezone
 
+from abx_plugins.plugins.base.url_cleaning import URL_REGEX, fix_url_from_markdown
+from abx_plugins.plugins.base.url_cleaning import sanitize_extracted_url as sanitize_plugin_url
+
 from .logging import COLOR_DICT
 
 
@@ -90,18 +93,6 @@ def ts_to_date_str(ts: Any) -> str | None:
 COLOR_REGEX = re.compile(r"\[(?P<arg_1>\d+)(;(?P<arg_2>\d+)(;(?P<arg_3>\d+))?)?m")
 
 
-# https://mathiasbynens.be/demo/url-regex
-URL_REGEX = re.compile(
-    r"(?=("
-    r"http[s]?://"  # start matching from allowed schemes
-    r"(?:[a-zA-Z]|[0-9]"  # followed by allowed alphanum characters
-    r"|[-_$@.&+!*\(\),]"  #   or allowed symbols (keep hyphen first to match literal hyphen)
-    r"|[^\u0000-\u007F])+"  #   or allowed unicode bytes
-    r'[^\]\[<>"\'\s]+'  # stop parsing at these symbols
-    r"))",
-    re.IGNORECASE | re.UNICODE,
-)
-
 # Maximum supported URL length. Very long URLs are rare but must be supported correctly
 # (e.g. data: URLs, deeply nested query strings). The Snapshot.url column is stored as a
 # variable-length TextField (so short URLs don't reserve space and very long URLs still
@@ -109,29 +100,6 @@ URL_REGEX = re.compile(
 # all keep working.
 MAX_URL_LENGTH = 65535
 
-QUOTE_DELIMITERS = (
-    '"',
-    "'",
-    "`",
-    "“",
-    "”",
-    "‘",
-    "’",
-)
-QUOTE_ENTITY_DELIMITERS = (
-    "&quot;",
-    "&#34;",
-    "&#x22;",
-    "&apos;",
-    "&#39;",
-    "&#x27;",
-)
-URL_ENTITY_REPLACEMENTS = (
-    ("&amp;", "&"),
-    ("&#38;", "&"),
-    ("&#x26;", "&"),
-)
-
 FILESIZE_UNITS: dict[str, int] = {
     "": 1,
     "b": 1,
@@ -153,36 +121,9 @@ FILESIZE_UNITS: dict[str, int] = {
 
 
 def sanitize_extracted_url(url: str) -> str:
-    """Trim quote garbage and dangling prose punctuation from an extracted URL candidate."""
-    cleaned = (url or "").strip()
-    if not cleaned:
-        return cleaned
-
-    lower_cleaned = cleaned.lower()
-    cut_index = len(cleaned)
-
-    for delimiter in QUOTE_DELIMITERS:
-        found_index = cleaned.find(delimiter)
-        if found_index != -1:
-            cut_index = min(cut_index, found_index)
-
-    for delimiter in QUOTE_ENTITY_DELIMITERS:
-        found_index = lower_cleaned.find(delimiter)
-        if found_index != -1:
-            cut_index = min(cut_index, found_index)
-
-    cleaned = cleaned[:cut_index].strip()
-    lower_cleaned = cleaned.lower()
-    for entity, replacement in URL_ENTITY_REPLACEMENTS:
-        while entity in lower_cleaned:
-            entity_index = lower_cleaned.find(entity)
-            cleaned = cleaned[:entity_index] + replacement + cleaned[entity_index + len(entity) :]
-            lower_cleaned = cleaned.lower()
-
-    cleaned = cleaned.rstrip(".,;:!?\\'\"")
-    cleaned = cleaned.rstrip('"')
-
-    return cleaned
+    """Use the standalone parser's quote rules, then trim dangling prose punctuation."""
+    cleaned = sanitize_plugin_url(url)
+    return cleaned.rstrip('.,;:!?\\"')
 
 
 def validate_url_length(url: str) -> str:
@@ -215,39 +156,6 @@ def parens_are_matched(string: str, open_char="(", close_char=")"):
     return count == 0
 
 
-def fix_url_from_markdown(url_str: str) -> str:
-    """
-    cleanup a regex-parsed url that may contain dangling trailing parens from markdown link syntax
-    helpful to fix URLs parsed from markdown e.g.
-      input:  https://wikipedia.org/en/some_article_(Disambiguation).html?abc=def).somemoretext
-      result: https://wikipedia.org/en/some_article_(Disambiguation).html?abc=def
-
-    IMPORTANT ASSUMPTION: valid urls wont have unbalanced or incorrectly nested parentheses
-    e.g. this will fail the user actually wants to ingest a url like 'https://example.com/some_wei)(rd_url'
-         in that case it will return https://example.com/some_wei (truncated up to the first unbalanced paren)
-    This assumption is true 99.9999% of the time, and for the rare edge case the user can use url_list parser.
-    """
-    trimmed_url = url_str
-    if len(trimmed_url) > 2048:
-        return trimmed_url
-
-    # cut off one trailing character at a time
-    # until parens are balanced e.g. /a(b)c).x(y)z -> /a(b)c
-    trim_attempts = 0
-    while trimmed_url and not parens_are_matched(trimmed_url) and trim_attempts < 256:
-        trimmed_url = trimmed_url[:-1]
-        trim_attempts += 1
-
-    if not trimmed_url or not parens_are_matched(trimmed_url):
-        return url_str
-
-    # make sure trimmed url is still valid
-    if any(match == trimmed_url for match in re.findall(URL_REGEX, trimmed_url)):
-        return trimmed_url
-
-    return url_str
-
-
 def split_comma_separated_urls(url: str):
     offset = 0
     while True:
@@ -274,7 +182,8 @@ def find_all_urls(urls_str: str):
         if match.start() in skipped_starts:
             continue
 
-        cleaned_match = sanitize_extracted_url(fix_url_from_markdown(match.group(1)))
+        cleaned_match = fix_url_from_markdown(match.group(1), preceding_text=urls_str[max(0, match.start(1) - 6) : match.start(1)])
+        cleaned_match = sanitize_extracted_url(cleaned_match)
         for offset, url in split_comma_separated_urls(cleaned_match):
             if offset:
                 skipped_starts.add(match.start() + offset)
@@ -388,9 +297,12 @@ def parse_date(date: Any) -> datetime | None:
         if date.tzinfo is None:
             return date.replace(tzinfo=timezone.utc)
 
-        offset = date.utcoffset()
-        assert offset == datetime.now(timezone.utc).utcoffset(), "Refusing to load a non-UTC date!"
-        return date
+        # Legacy SQLite ArchiveResult rows can retain explicit timezone offsets
+        # through migration. Django loads those as aware datetimes, whereas JSON
+        # imports reach the string branch below. Normalize both representations
+        # consistently so opening a migrated snapshot does not raise (#1889).
+        # Convert the instant; replacing tzinfo here would shift the capture time.
+        return date.astimezone(timezone.utc)
 
     if isinstance(date, (float, int)):
         date = str(date)

archivebox/opencode/__init__.py

@@ -1 +0,0 @@
-"""ArchiveBox OpenCode admin integration."""

archivebox/opencode/urls.py

@@ -1,13 +0,0 @@
-from django.urls import path, re_path
-
-from archivebox.opencode.views import agent_view, opencode_proxy_view
-
-
-urlpatterns = [
-    path("", agent_view, name="opencode-agent"),
-    re_path(
-        r"^opencode(?:/(?P<path>.*))?$",
-        opencode_proxy_view,
-        name="opencode-proxy",
-    ),
-]

archivebox/opencode/views.py

@@ -1,140 +0,0 @@
-"""Django adapter for the optional plugin; never import its runtime at startup."""
-
-import logging
-from io import BytesIO
-
-from asgiref.sync import sync_to_async
-
-from django.contrib.auth.middleware import AuthenticationMiddleware
-from django.contrib.auth.views import redirect_to_login
-from django.contrib.sessions.middleware import SessionMiddleware
-from django.core.exceptions import PermissionDenied
-from django.core.handlers.asgi import ASGIRequest
-from django.db import close_old_connections
-from django.http import Http404, HttpResponse, HttpResponseForbidden, StreamingHttpResponse
-from django.template import engines
-from django.urls import Resolver404, resolve
-from django.views.decorators.csrf import csrf_exempt
-
-from archivebox.config import CONSTANTS
-from archivebox.config.common import get_request_config
-from archivebox.core.middleware import ReverseProxyAuthMiddleware
-from archivebox.core.routes_util import build_admin_url, get_admin_host, get_api_base_url, get_base_url, host_matches
-from archivebox.plugins.discovery import get_plugin_template
-
-_LOGGER = logging.getLogger(__name__)
-
-
-def _runtime_settings(request, config):
-    from abx_plugins.plugins.opencode import runtime
-
-    settings = runtime._settings(config, CONSTANTS.DATA_DIR)
-    route_config = request.__dict__.get("archivebox_config")
-    settings.update(
-        archivebox_base_url=get_base_url(request=request, config=route_config).rstrip("/"),
-        archivebox_admin_url=build_admin_url("/admin/", request=request, config=route_config).rstrip("/"),
-        archivebox_api_url=f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/",
-    )
-    return runtime, settings
-
-
-def _dispatch(request, path=None):
-    try:
-        # Middleware already merged server defaults and live Machine overrides.
-        # Re-resolving every extractor's config here stalls the UI request burst.
-        config = get_request_config(request).model_dump(mode="json")
-        if not config.get("OPENCODE_ENABLED", False):
-            raise Http404
-        if not request.user.is_authenticated:
-            return redirect_to_login(request.get_full_path(), login_url="/admin/login/")
-        if not request.user.is_active or not request.user.is_superuser:
-            return HttpResponseForbidden("Agent access requires a superuser account.")
-
-        runtime, settings = _runtime_settings(request, config)
-        if path is None:
-            from archivebox.core.admin_site import archivebox_admin
-
-            context = {**archivebox_admin.each_context(request), **runtime.agent_context(settings)}
-            source = get_plugin_template("opencode", "agent", fallback=False)
-            if source is None:
-                raise RuntimeError("Agent template unavailable")
-            return HttpResponse(engines["django"].from_string(source).render(context, request))
-
-        if not runtime._origin_allowed(request.method, request.get_host(), request.headers):
-            return HttpResponseForbidden("Cross-origin agent requests are blocked.")
-        status, headers, body = runtime.proxy(
-            settings,
-            request.method,
-            path,
-            tuple((key, value) for key, values in request.GET.lists() for value in values),
-            request.headers,
-            request.body,
-        )
-        response_type = HttpResponse if isinstance(body, bytes) else StreamingHttpResponse
-        response = response_type(body, status=status, headers=headers)
-        response.xframe_options_exempt = True
-        response.headers["X-Frame-Options"] = "SAMEORIGIN"
-        response.headers["Content-Security-Policy"] = "frame-ancestors 'self'"
-        return response
-    except Http404:
-        raise
-    except Exception:
-        # Optional-service boundary, including imports and template rendering.
-        _LOGGER.exception("Optional AI service failed")
-        return HttpResponse("AI service unavailable. See server logs.", status=503, content_type="text/plain")
-
-
-def agent_view(request):
-    return _dispatch(request)
-
-
-@csrf_exempt
-def opencode_proxy_view(request, path=None):
-    return _dispatch(request, path=path or "")
-
-
-def _websocket_context(scope):
-    close_old_connections()
-    try:
-        match = resolve(scope["path"])
-        if match.url_name != "opencode-proxy":
-            raise PermissionDenied
-        request = ASGIRequest(
-            {**scope, "type": "http", "method": "GET", "scheme": "https" if scope.get("scheme") == "wss" else "http"},
-            BytesIO(),
-        )
-        route_config = get_request_config(request, resolve_plugins=False)
-        request.archivebox_config = route_config
-        if not route_config.CONTROL_PLANE_ENABLED:
-            raise PermissionDenied
-        if (
-            route_config.USES_SUBDOMAIN_ROUTING
-            and route_config.BASE_URL
-            and not host_matches(request.get_host(), get_admin_host(config=route_config, request=request))
-        ):
-            raise PermissionDenied
-        SessionMiddleware(_dispatch).process_request(request)
-        AuthenticationMiddleware(_dispatch).process_request(request)
-        ReverseProxyAuthMiddleware(_dispatch).process_request(request)
-        config = route_config.model_dump(mode="json")
-        if not config.get("OPENCODE_ENABLED") or not request.user.is_active or not request.user.is_superuser:
-            raise PermissionDenied
-        runtime, settings = _runtime_settings(request, config)
-        if not request.headers.get("Origin") or not runtime._origin_allowed("POST", request.get_host(), request.headers):
-            raise PermissionDenied
-        return runtime, settings, match.kwargs.get("path", "")
-    finally:
-        close_old_connections()
-
-
-async def websocket_view(scope, receive, send):
-    if (await receive())["type"] != "websocket.connect":
-        return
-    try:
-        runtime, settings, path = await sync_to_async(_websocket_context)(scope)
-        await runtime.websocket_proxy(settings, path, scope.get("query_string", b""), scope.get("subprotocols", []), receive, send)
-    except (PermissionDenied, Resolver404, Http404):
-        await send({"type": "websocket.close", "code": 1008})
-    except Exception:
-        _LOGGER.exception("Optional AI WebSocket failed")
-        await send({"type": "websocket.close", "code": 1011})

archivebox/personas/forms.py

@@ -122,14 +122,14 @@ class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm):
                 "Use the custom path/CDP option if the browser data lives elsewhere."
             )
 
-        self.build_plugin_groups(get_config(persona=self.instance) if self.instance and self.instance.pk else get_config())
+        self.build_plugin_groups(get_config(persona=self.instance) if not self.instance._state.adding else get_config())
 
     def clean_name(self) -> str:
         name = str(self.cleaned_data.get("name") or "").strip()
         is_valid, error_message = validate_persona_name(name)
         if not is_valid:
             raise forms.ValidationError(error_message)
-        return name
+        return self.instance.validate_name(name)
 
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()

archivebox/personas/models.py

@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any
 from collections.abc import Mapping
 
 from django.db import IntegrityError, models
+from django.core.exceptions import ValidationError
 from django.db.models.fields.json import KT
 from django.conf import settings
 from django.utils import timezone
@@ -74,8 +75,12 @@ def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, person
         derived["COOKIES_FILE"] = str(cookies_path)
 
     auth_path = persona_dir / "auth.json"
-    if "AUTH_STORAGE_FILE" not in derived and auth_path.exists():
-        derived["AUTH_STORAGE_FILE"] = str(auth_path)
+    if "AUTH_STORAGE_FILE" not in derived:
+        try:
+            if auth_path.stat().st_size > 0:
+                derived["AUTH_STORAGE_FILE"] = str(auth_path)
+        except FileNotFoundError:
+            pass
 
     derived["ACTIVE_PERSONA"] = name
     return derived
@@ -117,6 +122,13 @@ class Persona(ModelWithConfig):
         app_label = "personas"
 
     def save(self, *args, **kwargs):
+        # Adopt the spelling on disk, never rename a Chrome profile just to fix case.
+        self.__dict__.pop("_persona_path", None)
+        canonical_name = self.validate_name(self.name)
+        if canonical_name != self.name:
+            self.name = canonical_name
+            if kwargs.get("update_fields") is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*kwargs["update_fields"], "name"]))
         config = dict(self.config or {})
         if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
             from archivebox.config.common import get_config
@@ -133,10 +145,59 @@ class Persona(ModelWithConfig):
 
     @property
     def path(self) -> Path:
-        """Path to persona directory under PERSONAS_DIR."""
+        """Reuse the directory's spelling without renaming a Chrome profile."""
         from archivebox.config.constants import CONSTANTS
 
-        return CONSTANTS.PERSONAS_DIR / self.name
+        cached = self.__dict__.get("_persona_path")
+        if cached is None or cached[0] != self.name:
+            directory_name = self.directory_name(self.name)
+            cached = (self.name, CONSTANTS.PERSONAS_DIR / directory_name)
+            self.__dict__["_persona_path"] = cached
+        return cached[1]
+
+    @staticmethod
+    def directory_name(name: str) -> str:
+        from archivebox.config.constants import CONSTANTS
+        from archivebox.personas.importers import validate_persona_name
+
+        valid, error = validate_persona_name(name)
+        if not valid:
+            raise ValidationError(error)
+        # Linux can hold both Default/default; macOS usually aliases them. Reuse a
+        # single match on either filesystem, but never guess which conflicting
+        # profile contains the user's login state. Only list names, not profile trees.
+        try:
+            with os.scandir(CONSTANTS.PERSONAS_DIR) as entries:
+                matches = sorted(entry.name for entry in entries if entry.name.casefold() == name.casefold())
+        except FileNotFoundError:
+            matches = []
+        if len(matches) > 1:
+            raise ValidationError(
+                f"Conflicting persona directories: {'; '.join(str(CONSTANTS.PERSONAS_DIR / match) for match in matches)}. "
+                "Stop Chrome and rename the conflicting directories manually to distinct names; no profile directories were renamed.",
+            )
+        return matches[0] if matches else name
+
+    @classmethod
+    def find_named(cls, name: str) -> "Persona | None":
+        # SQLite's iexact is ASCII-only; Python casefold gives every entry point
+        # the same Unicode matching rules as the filesystem-name lookup above.
+        matches = [(pk, existing) for pk, existing in cls.objects.values_list("pk", "name") if existing.casefold() == name.casefold()]
+        if len(matches) > 1:
+            raise ValidationError(
+                f"Conflicting persona records: {'; '.join(f'{existing!r} (ID {pk})' for pk, existing in matches)}. "
+                "Rename the conflicting personas manually to distinct names; profile files have not been changed.",
+            )
+        return cls.objects.get(pk=matches[0][0]) if matches else None
+
+    def validate_name(self, name: str) -> str:
+        existing = self.find_named(name)
+        if existing is not None and existing.pk != self.pk:
+            raise ValidationError(
+                f"Name conflicts with persona {existing.name!r} (ID {existing.pk}, directory {existing.path}). "
+                "Choose a distinct name or rename that persona record manually.",
+            )
+        return self.directory_name(name)
 
     @property
     def CHROME_USER_DATA_DIR(self) -> str:
@@ -156,9 +217,12 @@ class Persona(ModelWithConfig):
 
     @property
     def AUTH_STORAGE_FILE(self) -> str:
-        """Derived path to auth.json for this persona (if it exists)."""
+        """Derived path to a nonempty auth.json export for this persona."""
         auth_path = self.path / "auth.json"
-        return str(auth_path) if auth_path.exists() else ""
+        try:
+            return str(auth_path) if auth_path.stat().st_size > 0 else ""
+        except FileNotFoundError:
+            return ""
 
     def get_derived_config(self) -> dict:
         """
@@ -171,10 +235,13 @@ class Persona(ModelWithConfig):
         - AUTH_STORAGE_FILE (derived from persona path, if file exists)
         - ACTIVE_PERSONA (set to this persona's name)
         """
-        return derive_persona_config(name=self.name, config=self.config, persona_dir=self.path)
+        return derive_persona_config(name=self.path.name, config=self.config, persona_dir=self.path)
 
     def ensure_dirs(self) -> None:
         """Create persona directories if they don't exist."""
+        # Recheck at profile preparation, including changes made outside this process.
+        self.__dict__.pop("_persona_path", None)
+        self.validate_name(self.name)
         self.path.mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_profile").mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_downloads").mkdir(parents=True, exist_ok=True)
@@ -246,16 +313,26 @@ class Persona(ModelWithConfig):
                     fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
 
     @classmethod
-    def get_or_create_named(cls, name: str) -> "Persona":
+    def get_or_create_named(cls, name: str, *, defaults: dict | None = None) -> "Persona":
+        # Init/startup must reuse a user-created 'default', not create a second
+        # DB row pointing at a different (or shared) profile (issue #1896).
         persona_name = (name or "Default").strip() or "Default"
-        persona = cls.objects.filter(name=persona_name).first()
+        persona = cls.find_named(persona_name)
         if persona is not None:
+            canonical_name = persona.validate_name(persona.name)
+            if canonical_name != persona.name:
+                persona.name = canonical_name
+                persona.save(update_fields=["name"])
             return persona
 
         try:
-            return cls.objects.create(name=persona_name)
-        except IntegrityError:
-            return cls.objects.get(name=persona_name)
+            return cls.objects.create(name=persona_name, **(defaults or {}))
+        except (IntegrityError, ValidationError):
+            persona = cls.find_named(persona_name)
+            if persona is None:
+                raise
+            persona.validate_name(persona.name)
+            return persona
 
     def runtime_root_for_crawl(self, crawl) -> Path:
         return Path(crawl.output_dir) / ".persona" / self.name
@@ -315,20 +392,9 @@ class Persona(ModelWithConfig):
             if chrome_binary:
                 (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # Hooks derive CHROME_USER_DATA_DIR/CHROME_DOWNLOADS_DIR from
-            # PERSONAS_DIR + ACTIVE_PERSONA. Point PERSONAS_DIR at the
-            # per-crawl runtime root here so CHROME_ISOLATION=crawl never
-            # leaks or reuses the template profile while keeping Chrome path
-            # derivation centralized in the Chrome plugin helpers.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        # Hooks derive Chrome profile paths from the runtime persona root.
+        # Use the same optional auth discovery as the template persona.
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
         crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
@@ -358,18 +424,7 @@ class Persona(ModelWithConfig):
         if chrome_binary:
             (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # See prepare_runtime_for_crawl(): snapshot isolation changes the
-            # persona root, not individual CHROME_* config keys, so standalone
-            # Chrome hooks and ArchiveBox-driven hooks resolve paths the same way.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def cleanup_runtime_for_crawl(self, crawl) -> None:
         shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True)

archivebox/plugins/discovery.py

@@ -2,7 +2,9 @@ __package__ = "archivebox.plugins"
 
 from collections.abc import Iterable
 from functools import lru_cache
-from pathlib import Path
+from importlib import import_module
+from pathlib import Path, PurePosixPath
+from types import ModuleType
 from typing import Any, Protocol, TypedDict
 
 from abx_plugins import get_plugins_dir
@@ -66,8 +68,143 @@ def get_plugin_name(plugin: str) -> str:
     """
     parts = plugin.split("_", 1)
     if len(parts) == 2 and parts[0].isdigit():
-        return parts[1]
-    return plugin
+        plugin = parts[1]
+    return get_archive_result_aliases().get(plugin, plugin)
+
+
+@lru_cache(maxsize=1)
+def get_archive_result_aliases() -> dict[str, str]:
+    aliases: dict[str, str] = {}
+    for plugin in get_plugin_catalog().values():
+        for alias in plugin.manifest.get("archive_result_aliases") or []:
+            alias = str(alias or "").strip()
+            if alias:
+                aliases[alias] = plugin.name
+    return aliases
+
+
+def get_archive_result_names(plugin_name: str) -> tuple[str, ...]:
+    """Return the canonical ArchiveResult name and its declared historic aliases."""
+    canonical = get_plugin_name(plugin_name)
+    aliases = get_archive_result_aliases()
+    return tuple(dict.fromkeys((canonical, plugin_name, *(alias for alias, target in aliases.items() if target == canonical))))
+
+
+def plugin_card_is_interactive(plugin_name: str) -> bool:
+    plugin = get_plugin_catalog().get(get_plugin_name(plugin_name))
+    return bool(plugin and plugin.manifest.get("card_interactive"))
+
+
+def get_plugin_output_extension_preference(plugin_name: str) -> tuple[tuple[str, ...], ...] | None:
+    plugin = get_plugin_catalog().get(get_plugin_name(plugin_name))
+    raw_groups = plugin.manifest.get("output_extension_preference") if plugin else None
+    if not isinstance(raw_groups, list):
+        return None
+    groups = []
+    for raw_group in raw_groups:
+        if not isinstance(raw_group, list):
+            continue
+        group = tuple(str(extension).lower() for extension in raw_group if str(extension).startswith("."))
+        if group:
+            groups.append(group)
+    return tuple(groups) or None
+
+
+def get_plugin_default_output_path(plugin_name: str) -> str | None:
+    prefix, separator, base_name = plugin_name.partition("_")
+    archive_result_name = base_name if separator and prefix.isdigit() else plugin_name
+    plugin = get_plugin_catalog().get(get_plugin_name(plugin_name))
+    raw_path = str(plugin.manifest.get("default_output_path") or "").strip() if plugin else ""
+    path = PurePosixPath(raw_path)
+    if not raw_path or raw_path.startswith("/") or ".." in path.parts or len(path.parts) != 1:
+        return None
+    return f"{archive_result_name}/{raw_path}"
+
+
+@lru_cache(maxsize=1)
+def get_snapshot_thumbnail_card_order() -> dict[str, int]:
+    """Return plugin-owned ArchiveResult names eligible for snapshot thumbnails."""
+    ordering: dict[str, int] = {}
+    for plugin in get_plugin_catalog().values():
+        declarations = plugin.manifest.get("snapshot_thumbnail_cards") or []
+        if not isinstance(declarations, list):
+            continue
+        for declaration in declarations:
+            if not isinstance(declaration, dict):
+                continue
+            result_name = str(declaration.get("archive_result_name") or "").strip()
+            order = declaration.get("order")
+            if not result_name or isinstance(order, bool) or not isinstance(order, int):
+                continue
+            ordering[result_name] = order
+    return ordering
+
+
+@lru_cache(maxsize=None)
+def get_snapshot_role_names(role: str) -> tuple[str, ...]:
+    """Return ArchiveResult names for a plugin-owned snapshot presentation role."""
+    manifest_key = f"snapshot_{role}"
+    names = []
+    for plugin in get_plugin_catalog().values():
+        if plugin.manifest.get(manifest_key):
+            names.extend(get_archive_result_names(plugin.name))
+    return tuple(dict.fromkeys(names))
+
+
+def get_snapshot_role_result(results, role: str):
+    """Select the first successful result declared for a presentation role."""
+    results_by_name = {result.plugin: result for result in results if result.status == "succeeded"}
+    return next((results_by_name[name] for name in get_snapshot_role_names(role) if name in results_by_name), None)
+
+
+@lru_cache(maxsize=1)
+def get_plugin_presentation_modules() -> tuple[ModuleType, ...]:
+    """Load optional plugin-owned presentation hooks declared in manifests."""
+    return tuple(module for plugin in get_plugin_catalog().values() if (module := get_plugin_presentation_module(plugin.name)) is not None)
+
+
+@lru_cache(maxsize=None)
+def get_plugin_presentation_module(plugin_name: str) -> ModuleType | None:
+    """Load one plugin's optional presentation hooks module."""
+    plugin = get_plugin_catalog().get(get_plugin_name(plugin_name))
+    module_name = str(plugin.manifest.get("presentation_module") or "").strip() if plugin else ""
+    return import_module(module_name) if module_name else None
+
+
+def plugin_has_custom_full_response(plugin_name: str) -> bool:
+    """Return whether a plugin renders its full template through a response hook."""
+    module = get_plugin_presentation_module(plugin_name)
+    return bool(module and getattr(module, "render_full_response", None))
+
+
+def get_extra_snapshot_output_cards(outputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Collect additional cards from plugin-owned presentation hooks."""
+    cards = []
+    for module in get_plugin_presentation_modules():
+        hook = getattr(module, "extra_snapshot_output_cards", None)
+        if hook is not None:
+            cards.extend(hook(outputs) or ())
+    return cards
+
+
+def serve_plugin_replay_asset(path: str, config, response_class):
+    """Give plugin presentation hooks the first chance to serve replay assets."""
+    for module in get_plugin_presentation_modules():
+        hook = getattr(module, "serve_replay_asset_response", None)
+        if hook is not None and (response := hook(path, config, response_class)) is not None:
+            from archivebox.misc.serve_static import add_replay_image_fallback
+
+            return add_replay_image_fallback(response)
+    return None
+
+
+def render_plugin_full_response(path: str, output_url: str, **kwargs):
+    """Let plugin presentation hooks render a saved output's full template."""
+    for module in get_plugin_presentation_modules():
+        render = getattr(module, "render_full_response", None)
+        if render is not None and (response := render(path, output_url, **kwargs)) is not None:
+            return response
+    return None
 
 
 def get_enabled_plugins(config: ConfigLookup | None = None, **config_kwargs: Any) -> list[str]:
@@ -151,12 +288,10 @@ def get_plugin_template(plugin: str, template_name: str, fallback: bool = True)
 
     Args:
         plugin: Plugin name (e.g., 'screenshot', '15_singlefile')
-        template_name: One of 'icon', 'card', 'full'
+        template_name: A template slot such as 'icon', 'card', or 'full'.
         fallback: If True, return default template if plugin template not found
     """
     base_name = get_plugin_name(plugin)
-    if base_name in ("yt-dlp", "youtube-dl"):
-        base_name = "ytdlp"
 
     catalog = get_plugin_catalog()
     if base_name in catalog:

archivebox/plugins/output_groups.py

@@ -1,74 +1,115 @@
 """Snapshot output taxonomy. Display preference is independent of hook order."""
 
-from pathlib import PurePosixPath
 from typing import Any
 
-from archivebox.plugins.discovery import get_plugin_name
+from archivebox.plugins.discovery import (
+    get_extra_snapshot_output_cards,
+    get_plugin_catalog,
+    get_plugin_name,
+    get_snapshot_thumbnail_card_order,
+)
 
-# Ordered tuples are preference lists; unordered groups sort by total output bytes.
-OUTPUT_GROUPS = (
-    ("html", "HTML", ("archivewebpage", "singlefile", "chrome_mhtml", "wget", "dom")),
-    ("raster", "Raster", ("screenshot", "pdf")),
-    ("article_text", "Article text", ("defuddle", "readability", "mercury", "trafilatura", "htmltotext")),
-    ("embedded_media", "Embedded media", ()),
-    ("metadata", "Metadata", ()),
-    ("other", "Other files", ()),
+_GROUP_LABELS = (
+    ("html", "HTML"),
+    ("raster", "Raster"),
+    ("article_text", "Article text"),
+    ("embedded_media", "Embedded media"),
+    ("metadata", "Metadata"),
+    ("other", "Other files"),
+)
+
+
+def _plugin_presentation() -> dict[str, dict[str, Any]]:
+    presentation = {}
+    for plugin in get_plugin_catalog().values():
+        manifest = plugin.manifest
+        order = manifest.get("snapshot_output_order")
+        presentation[plugin.name] = {
+            "group": manifest.get("snapshot_output_group") or "other",
+            "order": order if isinstance(order, int) and not isinstance(order, bool) else None,
+            "display_name": manifest.get("snapshot_display_name"),
+            "icon_hidden": bool(manifest.get("icon_hidden")),
+            "card_hidden": bool(manifest.get("card_hidden")),
+        }
+    return presentation
+
+
+PLUGIN_PRESENTATION = _plugin_presentation()
+OUTPUT_GROUPS = tuple(
+    (
+        group_id,
+        label,
+        tuple(
+            name
+            for name, metadata in sorted(
+                PLUGIN_PRESENTATION.items(),
+                key=lambda item: (item[1]["order"] is None, item[1]["order"] or 0, item[0]),
+            )
+            if metadata["group"] == group_id and metadata["order"] is not None
+        ),
+    )
+    for group_id, label in _GROUP_LABELS
 )
 UNORDERED_OUTPUTS = {
-    "embedded_media": {"responses", "ytdlp", "yt-dlp", "youtube-dl", "gallerydl", "forumdl", "git", "media", "liteparse"},
-    "metadata": {
-        "dns",
-        "headers",
-        "seo",
-        "accessibility",
-        "redirects",
-        "consolelog",
-        "sslcerts",
-        "ssl",
-        "title",
-        "favicon",
-        "hashes",
-        "chrome",
-        "parse_html_urls",
-        "parse_txt_urls",
-        "parse_dom_outlinks",
-        "parse_jsonl_urls",
-        "parse_netscape_urls",
-        "parse_rss_urls",
-        "tlsnotary",
-        "opentimestamps",
-    },
+    group_id: {name for name, metadata in PLUGIN_PRESENTATION.items() if metadata["group"] == group_id and metadata["order"] is None}
+    for group_id, _label in _GROUP_LABELS
 }
+HIDDEN_ICON_PLUGINS = frozenset(name for name, metadata in PLUGIN_PRESENTATION.items() if metadata["icon_hidden"])
+HIDDEN_CARD_PLUGINS = frozenset(name for name, metadata in PLUGIN_PRESENTATION.items() if metadata["card_hidden"])
 
 
-def _responses_html_card(outputs: list[dict[str, Any]]) -> dict[str, Any] | None:
-    # Presentation-only exception to the usual 1:1 plugin:card rule. Responses
-    # owns both this HTML view and its media gallery; responses_html is only a
-    # unique card/anchor ID, never a plugin or a synthetic ArchiveResult.
-    # Refactor this exception if plugins gain support for multiple cards.
-    for output in outputs:
-        if output["name"] != "responses" or not (result := output.get("result")):
+def output_group_for_plugin(plugin: str) -> str:
+    """Return the snapshot-detail stack that contains a plugin output."""
+    plugin = get_plugin_name(plugin)
+    for group_id, _, plugins in OUTPUT_GROUPS:
+        if plugin in plugins or plugin in UNORDERED_OUTPUTS.get(group_id, set()):
+            return group_id
+    return "other"
+
+
+def plugin_icon_is_hidden(plugin: str) -> bool:
+    return get_plugin_name(plugin) in HIDDEN_ICON_PLUGINS
+
+
+def plugin_card_is_hidden(plugin: str) -> bool:
+    return get_plugin_name(plugin) in HIDDEN_CARD_PLUGINS
+
+
+def display_plugin_name(plugin: str) -> str:
+    canonical = get_plugin_name(plugin)
+    return PLUGIN_PRESENTATION.get(canonical, {}).get("display_name") or plugin.replace("_", " ").replace("-", " ").title()
+
+
+def plugin_output_sizes(results) -> dict[str, int]:
+    """Count each plugin's files once, even when several hooks report them."""
+    files_by_plugin: dict[str, dict[str, int]] = {}
+    for result in results:
+        files = files_by_plugin.setdefault(result.plugin, {})
+        for path, metadata in result.output_file_map().items():
+            files[path] = result._coerce_output_file_size(metadata.get("size"))
+    return {plugin: sum(files.values()) for plugin, files in files_by_plugin.items() if files}
+
+
+def order_output_plugins(plugins, sizes: dict[str, int]) -> list[str]:
+    """Use the detail-card ordering for icon piles, without synthetic cards."""
+    return [output["name"] for output in order_snapshot_outputs([{"name": plugin, "size": sizes.get(plugin, 0)} for plugin in plugins])]
+
+
+def snapshot_thumbnail_result(results):
+    """Select the first available card using plugin-owned thumbnail priority."""
+    card_order = get_snapshot_thumbnail_card_order()
+    candidates = []
+    for result in results:
+        if result.plugin not in card_order:
             continue
-        path = str(output.get("path") or "")
-        if not path.startswith("responses/") or ".." in PurePosixPath(path).parts:
+        try:
+            path = result.embed_path()
+        except (TypeError, ValueError, OverflowError):
             continue
-        files = result.output_file_map()
-        info = files.get(path) or files.get(path.removeprefix("responses/")) or {}
-        mime = str(info.get("mimetype") or "").split(";")[0].lower()
-        if not info or result._coerce_output_file_size(info.get("size")) <= 0:
+        if not path:
             continue
-        if mime not in {"text/html", "application/xhtml+xml"} and PurePosixPath(path).suffix.lower() not in {".html", ".htm"}:
-            continue
-        return {
-            "name": "responses_html",
-            "path": path,
-            "folder_path": "responses",
-            "ts": output.get("ts"),
-            "size": 0,  # Bytes are already counted by the real Responses card.
-            "result": None,
-            "direct_preview_path": f"{path}?card=responses_html",
-        }
-    return None
+        candidates.append((card_order[result.plugin], str(result.id), result))
+    return min(candidates, default=(None, None, None))[2]
 
 
 def order_snapshot_outputs(outputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
@@ -82,14 +123,18 @@ def order_snapshot_outputs(outputs: list[dict[str, Any]]) -> list[dict[str, Any]
         for plugin in plugins:
             preferences[plugin] = (group_id, None)
 
-    if card := _responses_html_card(outputs):
-        outputs = [*outputs, card]
-    preferences["responses_html"] = ("html", len(OUTPUT_GROUPS[0][2]))
+    if cards := get_extra_snapshot_output_cards(outputs):
+        outputs = [*outputs, *cards]
 
     def sort_key(output):
         plugin = get_plugin_name(output["name"])
-        group_id, rank = preferences.get(plugin, ("other", None))
+        declared_group = output.get("output_group")
+        declared_order = output.get("output_order")
+        if declared_group in group_order and isinstance(declared_order, int) and not isinstance(declared_order, bool):
+            group_id, rank = declared_group, declared_order
+        else:
+            group_id, rank = preferences.get(plugin, ("other", None))
         output["output_group"] = group_id
-        return (group_order[group_id], rank if rank is not None else -int(output.get("size") or 0), plugin)
+        return (group_order[group_id], rank is None, rank if rank is not None else -int(output.get("size") or 0), plugin)
 
     return sorted(outputs, key=sort_key)

archivebox/progressmonitor/collection.py

@@ -1,4 +1,15 @@
-"""Non-blocking, database-only collection totals for the progress monitor."""
+"""Cached display totals for progress polling and the public snapshot list.
+
+Missing/stale statistics are intentional: returning useful rows and progress
+quickly matters more than exact initial counts. Each viewer may poll once per
+second; repeating aggregates per request would consume the small server's CPU
+and hold SQLite read locks needed by the archiving runner. Callers must accept
+None on a cold cache and must never wait for the refresh or count synchronously.
+
+Background reads still take normal SQLite read locks. Existing indexes, short
+batches where needed, and throttled refreshes limit their cost; moving work to
+a thread does not make a scan free or require switching the database to WAL.
+"""
 
 import logging
 from threading import Lock, Thread
@@ -9,14 +20,77 @@ from django.db import close_old_connections, connections
 from django.db.models import Count, Sum
 
 
+# This is only a process-local gate for background work, not a DB/file lock.
+# Every request tries it without waiting; a busy refresh means returning the
+# cached value (or None). Serializing refreshes also avoids competing scans for
+# different crawls/users when many viewer tabs open at once.
 _refresh_lock = Lock()
 
 
-def collection_summary(user):
-    user_id = None if user.is_superuser else user.pk
-    key = f"progress-collection:{'all' if user_id is None else user_id}"
+def crawl_summary(crawl_id, *, snapshot_id=None):
+    """Sample sealed snapshot metadata without adding table scans to polls."""
+    key = f"progress-crawl:{crawl_id}:{snapshot_id or 'all'}"
     summary = cache.get(key)
-    # Even an indexed SUM scans the index. Never make an HTTP request wait for
+    if (summary is None or time() - summary["sampled_at"] >= 30) and _refresh_lock.acquire(blocking=False):
+        if not cache.add(f"{key}:refresh", True, timeout=30):
+            _refresh_lock.release()
+            return summary
+
+        def refresh():
+            try:
+                close_old_connections()
+                from archivebox.core.models import Snapshot
+
+                snapshots = Snapshot.objects.filter(crawl_id=crawl_id, status=Snapshot.StatusChoices.SEALED)
+                if snapshot_id is not None:
+                    snapshots = snapshots.filter(pk=snapshot_id)
+                snapshots = snapshots.order_by("modified_at").values_list("modified_at", "downloaded_at", "output_size")
+                totals = {"snapshots": 0, "cancelled": 0, "bytes": 0}
+                modified_at, offset = None, 0
+                while True:
+                    batch_scope = snapshots.filter(modified_at__gte=modified_at) if modified_at is not None else snapshots
+                    # Use (crawl, status, modified_at), releasing the SQLite read
+                    # cursor after each small batch. Never hold a cursor over the
+                    # whole crawl, even in this background worker.
+                    batch = list(batch_scope[offset : offset + 100])
+                    for _, downloaded_at, size in batch:
+                        totals["snapshots"] += 1
+                        totals["cancelled"] += downloaded_at is None
+                        totals["bytes"] += size or 0
+                    if len(batch) < 100:
+                        break
+                    last_modified_at = batch[-1][0]
+                    # Only offset within ties; otherwise advance the indexed
+                    # timestamp range instead of rescanning previous pages.
+                    offset = (offset if last_modified_at == modified_at else 0) + sum(row[0] == last_modified_at for row in batch)
+                    modified_at = last_modified_at
+                totals["sampled_at"] = time()
+                cache.set(key, totals, timeout=3600)
+            except Exception:
+                logging.getLogger(__name__).exception("Could not refresh crawl totals")
+            finally:
+                connections.close_all()
+                _refresh_lock.release()
+
+        try:
+            Thread(target=refresh, name="crawl-summary", daemon=True).start()
+        except RuntimeError:
+            _refresh_lock.release()
+            logging.getLogger(__name__).exception("Could not start crawl totals refresh")
+    return summary
+
+
+def collection_summary(user, *, public=False):
+    """Return cached totals immediately and, when due, start one refresh.
+
+    Public pages need no extra AJAX/polling for these totals: an ordinary later
+    page load reads the populated cache. Keep this best-effort display cache
+    separate from pagination/authorization decisions, which need actual rows.
+    """
+    user_id = None if user.is_superuser else user.pk
+    key = "progress-collection:public" if public else f"progress-collection:{'all' if user_id is None else user_id}"
+    summary = cache.get(key)
+    # Even indexed COUNT/SUM aggregates scan the index. Never make HTTP wait for
     # that scan on a large collection. Serve stale totals while one worker refreshes;
     # a single non-blocking lock prevents a queue of scans from concurrent clients.
     if (summary is None or time() - summary["sampled_at"] >= 60) and _refresh_lock.acquire(blocking=False):
@@ -32,13 +106,29 @@ def collection_summary(user):
                 from archivebox.core.models import Snapshot
 
                 snapshots = Snapshot.objects.all()
-                if user_id is not None:
-                    snapshots = snapshots.filter(crawl__created_by_id=user_id)
-                # COUNT(*) + SUM(output_size) can use Snapshot's covering size index.
-                # Snapshot.output_size is maintained when ArchiveResult sizes change.
-                totals = snapshots.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
-                totals["bytes"] = totals["bytes"] or 0
+                if public:
+                    from archivebox.core.permissions import PERMISSIONS_PUBLIC
+
+                    # SQLite can choose a table scan for equality after ANALYZE
+                    # on an all-public archive. Equal range bounds keep the count
+                    # on the existing permissions index. It is still O(n), so
+                    # public pages share this background cache, never await it.
+                    # Count the indexed, non-null permissions column so this
+                    # needs no Snapshot table reads or new covering-index migration.
+                    totals = snapshots.filter(permissions__gte=PERMISSIONS_PUBLIC, permissions__lte=PERMISSIONS_PUBLIC).aggregate(
+                        snapshots=Count("permissions"),
+                    )
+                else:
+                    if user_id is not None:
+                        snapshots = snapshots.filter(crawl__created_by_id=user_id)
+                    # COUNT(*) + SUM(output_size) can use Snapshot's covering size index.
+                    # Snapshot.output_size is maintained when ArchiveResult sizes change.
+                    totals = snapshots.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
+                totals["bytes"] = totals.get("bytes") or 0
                 totals["sampled_at"] = time()
+                # Retain old totals longer than the 60s refresh interval so a
+                # slow/failed refresh does not force every tab back to a cold
+                # count. Staleness is acceptable; blocking page loads is not.
                 cache.set(key, totals, timeout=3600)
             except Exception:
                 logging.getLogger(__name__).exception("Could not refresh collection totals")

archivebox/progressmonitor/templates/progressmonitor/progress_monitor.html

@@ -549,6 +549,7 @@
         display: flex;
         align-items: center;
         justify-content: center;
+        position: relative;
         width: 84px;
         height: 52px;
         flex: 0 0 84px;
@@ -570,6 +571,17 @@
         object-position: top center;
         display: block;
     }
+    #progress-monitor .snapshot-preview img:not([data-loaded]),
+    #progress-monitor .snapshot-favicon:not([data-loaded]) {
+        visibility: hidden;
+    }
+    #progress-monitor .snapshot-preview-loading {
+        position: absolute;
+        font-size: 20px;
+    }
+    #progress-monitor .snapshot-preview img[data-loaded] + .snapshot-preview-loading {
+        display: none;
+    }
     #progress-monitor .snapshot-preview.placeholder {
         font-size: 20px;
     }
@@ -580,12 +592,17 @@
     #progress-monitor .snapshot-title-line {
         display: flex;
         align-items: center;
-        gap: 7px;
+        position: relative;
+        padding-left: 23px;
         min-width: 0;
     }
     #progress-monitor .snapshot-favicon {
         width: 16px;
         height: 16px;
+        position: absolute;
+        left: 0;
+        top: 50%;
+        transform: translateY(-50%);
         border-radius: 3px;
         object-fit: contain;
         background: #fff;
@@ -1061,54 +1078,133 @@
     }
     function stableSnapshotMedia(snapshot) {
         // The progress endpoint can briefly omit media URLs while ArchiveResult
-        // rows settle. Once this page has seen a snapshot image URL, keep using
-        // it so the preview/favicon branch does not flicker back to placeholder.
+        // rows settle. Keep displaying the last *loaded* image while probing
+        // new URLs offscreen: an output URL may 404 until its capture finishes.
         const snapshotId = String(snapshot.id || '');
         const media = snapshotMedia.get(snapshotId) || {};
-        if (snapshot.favicon_url) media.favicon_url ||= snapshot.favicon_url;
+        if (snapshot.favicon_url && snapshot.favicon_url !== media.favicon_url) {
+            media.favicon_url = snapshot.favicon_url;
+            media.favicon_next_probe = 0;
+            media.favicon_retry_delay = 5000;
+        }
         if (snapshot.preview_url) {
-            media.preview_url ||= snapshot.preview_url;
-            media.preview_link ||= snapshot.preview_link;
-            media.preview_fallbacks ||= snapshot.preview_fallbacks || [];
+            if (snapshot.preview_url !== media.preview_url) {
+                media.preview_url = snapshot.preview_url;
+                media.preview_next_probe = 0;
+                media.preview_retry_delay = 5000;
+            }
+            media.preview_link = snapshot.preview_link || media.preview_link;
         }
         if (snapshotId) snapshotMedia.set(snapshotId, media);
+        probeSnapshotImage(media, 'favicon');
+        probeSnapshotImage(media, 'preview');
         return media;
     }
-    function preserveStableMediaNodes(nextRoot) {
-        // Snapshot preview/favicon files are immutable once visible. Keep the
-        // loaded DOM nodes across poll renders so the browser never reloads
-        // them while progress text/badges continue updating around them.
-        const currentCards = new Map(
-            Array.from(crawlTree.querySelectorAll('.snapshot-item[data-snapshot-id]')).map(card => [card.dataset.snapshotId, card])
-        );
-        nextRoot.querySelectorAll('.snapshot-item[data-snapshot-id]').forEach(nextCard => {
-            const currentCard = currentCards.get(nextCard.dataset.snapshotId);
-            if (!currentCard) return;
-            currentCard.querySelectorAll('[data-stable-media]').forEach(currentNode => {
-                const nextNode = nextCard.querySelector(`[data-stable-media="${currentNode.dataset.stableMedia}"]`);
-                if (nextNode) nextNode.replaceWith(currentNode);
-            });
-        });
+    function probeSnapshotImage(media, kind) {
+        const url = media[`${kind}_url`];
+        if (!url || media[`${kind}_ready_url`] === url || media[`${kind}_probing`] === url || Date.now() < (media[`${kind}_next_probe`] || 0)) return;
+        media[`${kind}_probing`] = url;
+        const probe = new Image();
+        probe.onload = function() {
+            if (media[`${kind}_url`] !== url) return;
+            media[`${kind}_ready_url`] = url;
+            media[`${kind}_ready_src`] = probe.src;
+            media[`${kind}_probing`] = '';
+            media[`${kind}_retry_delay`] = 5000;
+        };
+        probe.onerror = function() {
+            if (media[`${kind}_url`] !== url) return;
+            media[`${kind}_probing`] = '';
+            const delay = media[`${kind}_retry_delay`] || 5000;
+            media[`${kind}_next_probe`] = Date.now() + delay;
+            media[`${kind}_retry_delay`] = Math.min(delay * 2, 30000);
+        };
+        // A previous 404 may be cached. Bust only the background probe URL;
+        // the visible card keeps its loaded image or fixed-size placeholder.
+        const probeUrl = new URL(url, window.location.href);
+        probeUrl.searchParams.set('progress_retry', String(Date.now()));
+        probe.src = probeUrl.href;
+    }
+    function sameProgressNode(current, next) {
+        if (current.nodeType !== next.nodeType) return false;
+        if (current.nodeType !== Node.ELEMENT_NODE) return true;
+        if (current.tagName !== next.tagName) return false;
+        for (const key of ['crawlId', 'snapshotId', 'stableMedia']) {
+            if ((current.dataset[key] || next.dataset[key]) && current.dataset[key] !== next.dataset[key]) return false;
+        }
+        return (current.className || '').split(' ')[0] === (next.className || '').split(' ')[0];
+    }
+    function reconcileProgressNode(current, next) {
+        if (current.nodeType === Node.TEXT_NODE) {
+            if (current.textContent !== next.textContent) current.textContent = next.textContent;
+            return;
+        }
+        if (current.nodeType !== Node.ELEMENT_NODE) return;
+        // Never detach a loaded image on a poll. Its URL only changes when the
+        // progress API reports a new output; keep a successful fallback too.
+        if (current.dataset.stableMedia && current.dataset.mediaUrl === next.dataset.mediaUrl) {
+            if (current.tagName === 'A' && current.getAttribute('href') !== next.getAttribute('href')) {
+                current.setAttribute('href', next.getAttribute('href'));
+            }
+            return;
+        }
+        for (const attr of Array.from(current.attributes)) {
+            if (!next.hasAttribute(attr.name)) current.removeAttribute(attr.name);
+        }
+        for (const attr of Array.from(next.attributes)) {
+            if (current.getAttribute(attr.name) !== attr.value) current.setAttribute(attr.name, attr.value);
+        }
+        reconcileProgressChildren(current, next);
+    }
+    function reconcileProgressChildren(current, next) {
+        let position = 0;
+        const nextNodes = Array.from(next.childNodes);
+        for (const nextChild of nextNodes) {
+            let child = current.childNodes[position];
+            if (!child) {
+                current.appendChild(nextChild);
+            } else if (sameProgressNode(child, nextChild)) {
+                reconcileProgressNode(child, nextChild);
+            } else {
+                const laterNext = nextNodes.slice(position + 1);
+                if (laterNext.some(node => sameProgressNode(child, node))) {
+                    // A new optional node (such as a favicon) precedes the old
+                    // one. Insert it without detaching the old loaded image.
+                    current.insertBefore(nextChild, child);
+                } else if (Array.from(current.childNodes).slice(position + 1).some(node => sameProgressNode(node, nextChild))) {
+                    // The old optional node disappeared; remove only that node.
+                    child.remove();
+                    reconcileProgressNode(current.childNodes[position], nextChild);
+                } else {
+                    child.replaceWith(nextChild);
+                }
+            }
+            position += 1;
+        }
+        while (current.childNodes.length > position) current.lastChild.remove();
     }
     function replaceCrawlTree(html) {
         const template = document.createElement('template');
         template.innerHTML = html;
-        preserveStableMediaNodes(template.content);
-        crawlTree.replaceChildren(...template.content.childNodes);
+        reconcileProgressChildren(crawlTree, template.content);
     }
-    window.nextPreviewFallback = function(img) {
-        const fallbacks = (img.dataset.fallbacks || '').split(',').filter(Boolean);
-        if (fallbacks.length > 0) {
-            img.src = fallbacks.shift();
-            img.dataset.fallbacks = fallbacks.join(',');
-        } else {
-            const preview = img.closest('.snapshot-preview');
-            if (preview) {
-                preview.removeAttribute('data-stable-media');
-                preview.classList.add('placeholder');
-                preview.innerHTML = '<span>▤</span>';
+    window.failedProgressImage = function(img) {
+        const snapshotId = img.closest('.snapshot-item')?.dataset.snapshotId;
+        const media = snapshotMedia.get(snapshotId);
+        if (media) {
+            for (const kind of ['favicon', 'preview']) {
+                if (media[`${kind}_ready_src`] === img.src) {
+                    media[`${kind}_ready_url`] = '';
+                    media[`${kind}_ready_src`] = '';
+                    media[`${kind}_next_probe`] = 0;
+                }
             }
         }
+        const preview = img.closest('.snapshot-preview');
+        if (preview) {
+            preview.classList.add('placeholder');
+            preview.innerHTML = '<span>▤</span>';
+        } else img.remove();
     };
     function formatUrl(url) {
         if (!url) return '(no URL)';
@@ -1466,11 +1562,13 @@
         const titleText = snapshot.title || formatUrl(snapshot.full_url || snapshot.url);
         const urlText = snapshot.full_url || snapshot.url || '';
         const media = stableSnapshotMedia(snapshot);
-        const faviconHtml = media.favicon_url
-            ? `<img class="snapshot-favicon" data-stable-media="favicon" src="${escapeAttr(media.favicon_url)}" alt="" decoding="async" loading="lazy" onerror="this.remove()">`
+        const faviconHtml = media.favicon_ready_src
+            ? `<img class="snapshot-favicon" data-stable-media="favicon" data-media-url="${escapeAttr(media.favicon_ready_url)}" src="${escapeAttr(media.favicon_ready_src)}" alt="" decoding="async" loading="lazy" onload="this.dataset.loaded='1'" onerror="failedProgressImage(this)">`
             : '';
-        const previewHtml = media.preview_url
-            ? `<a class="snapshot-preview" data-stable-media="preview" href="${escapeAttr(media.preview_link || snapshot.view_url || adminUrl)}" title="Open snapshot output"><img src="${escapeAttr(media.preview_url)}" alt="" decoding="async" loading="lazy" data-fallbacks="${escapeAttr((media.preview_fallbacks || []).join(','))}" onerror="nextPreviewFallback(this)"></a>`
+        const previewSrc = media.preview_ready_src || media.favicon_ready_src;
+        const previewMediaUrl = media.preview_ready_src ? media.preview_ready_url : media.favicon_ready_url;
+        const previewHtml = previewSrc
+            ? `<a class="snapshot-preview" data-stable-media="preview" data-media-url="${escapeAttr(previewMediaUrl)}" href="${escapeAttr(media.preview_link || snapshot.view_url || adminUrl)}" title="Open snapshot output"><img src="${escapeAttr(previewSrc)}" alt="" decoding="async" loading="lazy" onload="this.dataset.loaded='1'" onerror="failedProgressImage(this)"><span class="snapshot-preview-loading">▤</span></a>`
             : `<a class="snapshot-preview placeholder" href="${escapeAttr(snapshot.view_url || adminUrl)}" title="Open snapshot"><span>${statusIcon}</span></a>`;
 
         let extractorHtml = '';
@@ -1806,12 +1904,22 @@
     const progressEndpoint = monitor.dataset.progressEndpoint || '/progress.json';
 
     function fetchProgress() {
+        // Opening a replay replaces the queued snapshot's progress panel.
+        // Its interval and already-running fetch must stop using that old DOM.
+        if (!monitor.isConnected) {
+            stopPolling();
+            return;
+        }
         fetch(progressEndpoint, { credentials: 'same-origin' })
             .then(response => {
                 if (!response.ok) throw new Error(`HTTP ${response.status}`);
                 return response.json();
             })
             .then(data => {
+                if (!monitor.isConnected) {
+                    stopPolling();
+                    return;
+                }
                 if (data.error) {
                     console.error('Progress API error:', data.error, data.traceback);
                     setOrchestratorState('error', 'Backend not responding');
@@ -1823,6 +1931,10 @@
                 updateProgress(data);
             })
             .catch(error => {
+                if (!monitor.isConnected) {
+                    stopPolling();
+                    return;
+                }
                 console.error('Progress fetch error:', error);
                 setOrchestratorState('error', 'Backend not responding');
                 idleMessage.textContent = 'Fetch Error: ' + error.message;
@@ -1831,7 +1943,7 @@
     }
 
     function startPolling() {
-        if (pollInterval) return;
+        if (pollInterval || !monitor.isConnected) return;
         fetchProgress();
         pollInterval = setInterval(fetchProgress, pollDelayMs);
     }
@@ -1935,7 +2047,13 @@
 
     // Start polling when page loads
     startPolling();
-    setInterval(updateDurationBadges, 1000);
+    const durationInterval = setInterval(() => {
+        if (!monitor.isConnected) {
+            clearInterval(durationInterval);
+            return;
+        }
+        updateDurationBadges();
+    }, 1000);
 
     // Pause polling when tab is hidden
     document.addEventListener('visibilitychange', function() {

archivebox/progressmonitor/views.py

@@ -7,17 +7,15 @@ from typing import Literal
 from abx_dl.events import PROCESS_EXIT_SKIPPED
 from django.conf import settings
 from django.db import DatabaseError
-from django.db.models import CharField, Count, Q, Sum
+from django.db.models import CharField, Count, Q
 from django.db.models.functions import Cast
 from django.http import HttpResponse, JsonResponse
 from django.utils import timezone
 
-from archivebox.config import CONSTANTS
-from archivebox.config.common import get_config
 from archivebox.core.permissions import can_view_snapshot, is_admin_user
-from archivebox.core.routes_util import build_snapshot_url, build_web_url, get_api_base_url
+from archivebox.core.routes_util import build_snapshot_detail_url, build_snapshot_url, get_api_base_url
 from archivebox.misc.logging_util import printable_filesize
-from archivebox.plugins.discovery import discover_plugin_configs
+from archivebox.plugins.discovery import discover_plugin_configs, get_plugin_default_output_path, get_snapshot_role_result
 
 
 def progress_endpoint(scope: Literal["crawl", "snapshot"] | None = None, object_id: object | None = None) -> str:
@@ -152,68 +150,25 @@ def live_progress_view(request, *, authorized_snapshot=None):
 
             return hook_details(Path(hook_path).name, plugin=Path(hook_path).parent.name or "setup")
 
-        def archiveresult_output_path(ar) -> str | None:
-            output_file_map = ar.output_files if isinstance(ar.output_files, dict) else {}
-
-            def is_root_relative(path: str) -> bool:
-                metadata = output_file_map.get(path) or {}
-                return bool(isinstance(metadata, dict) and metadata.get("root_relative"))
-
-            if ar.output_str:
-                raw_output = str(ar.output_str).strip()
-                if ar._looks_like_output_path(raw_output, ar.plugin):
-                    output_path = Path(raw_output)
-                    if output_path.is_absolute():
-                        return None
-
-                    if raw_output.startswith(f"{ar.plugin}/"):
-                        candidates = [raw_output]
-                    elif len(output_path.parts) == 1:
-                        candidates = [f"{ar.plugin}/{raw_output}", raw_output]
-                    else:
-                        candidates = [raw_output]
-
-                    if raw_output in output_file_map and is_root_relative(raw_output):
-                        return raw_output
-
-                    for relative_path in candidates:
-                        plugin_relative = relative_path.removeprefix(f"{ar.plugin}/")
-                        if relative_path in output_file_map:
-                            return f"{ar.plugin}/{relative_path}" if not relative_path.startswith(f"{ar.plugin}/") else relative_path
-                        if plugin_relative in output_file_map:
-                            return f"{ar.plugin}/{plugin_relative}"
-
-            output_file_paths = list(output_file_map.keys())
-            if output_file_paths:
-                fallback_path = ArchiveResult._fallback_output_file_path(output_file_paths, ar.plugin, output_file_map)
-                if fallback_path:
-                    if is_root_relative(fallback_path):
-                        return fallback_path
-                    return f"{ar.plugin}/{fallback_path}"
-
-            return None
-
         def snapshot_output_url(snapshot, output_path: str) -> str:
             return build_snapshot_url(str(snapshot["id"]), output_path, request=request, config=request_config)
 
         def snapshot_archive_path(snapshot) -> str:
-            if snapshot["fs_version"] in ("0.7.0", "0.8.0"):
-                return f"{CONSTANTS.ARCHIVE_DIR_NAME}/{snapshot['timestamp']}"
             crawl = crawls_by_id.get(str(snapshot["crawl_id"]))
-            username = "web"
-            if crawl is not None and crawl["created_by_id"]:
-                username = crawl["created_by__username"]
-            if username == "system":
-                username = "web"
-            date_base = snapshot["bookmarked_at"] or snapshot["created_at"]
-            date_str = date_base.strftime("%Y%m%d") if date_base else "unknown"
-            domain = Snapshot.extract_domain_from_url(snapshot["url"])
-            return f"{username}/{date_str}/{domain}/{snapshot['id']}"
+            return Snapshot.archive_path_for_values(
+                fs_version=snapshot["fs_version"],
+                timestamp=snapshot["timestamp"],
+                snapshot_id=snapshot["id"],
+                url=snapshot["url"],
+                username=crawl["created_by__username"] if crawl is not None and crawl["created_by_id"] else "web",
+                bookmarked_at=snapshot["bookmarked_at"],
+                created_at=snapshot["created_at"],
+            )
 
         def snapshot_view_url(snapshot, output_path: str = "") -> str:
-            anchor = f"#{output_path}" if output_path else ""
-            return build_web_url(
-                f"/{snapshot_archive_path(snapshot)}/index.html{anchor}",
+            return build_snapshot_detail_url(
+                snapshot_archive_path(snapshot),
+                output_path=output_path,
                 request=request,
                 config=request_config,
             )
@@ -226,53 +181,37 @@ def live_progress_view(request, *, authorized_snapshot=None):
 
         running_screencasts = {}
 
-        def screencast_frame_url(crawl_id: str, crawl_dir: Path) -> str:
-            frame_path = crawl_dir / "chrome_screencast" / "latest.jpg"
-            try:
-                frame_stat = frame_path.stat()
-            except OSError:
-                return ""
-            if frame_stat.st_size <= 0:
-                return ""
+        def screencast_frame_url(crawl_id: str) -> str:
             started_at = running_screencasts.get(crawl_id)
-            if started_at is None or frame_stat.st_mtime < started_at.timestamp():
+            if started_at is None:
                 return ""
-            rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}"
+            # File existence/freshness belongs to the image request, never to
+            # the one-second JSON poll (archive storage may be remote).
+            rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={int(now.timestamp())}&after={started_at.timestamp()}"
             return f"{api_base}{rel}" if api_base else rel
 
-        machine_id = Machine.current().id
+        machine = Machine.current_readonly()
+        machine_id = machine.id if machine is not None else None
         orchestrator_proc = (
             Process.objects.filter(
                 machine_id=machine_id,
                 process_type=Process.TypeChoices.ORCHESTRATOR,
                 status=Process.StatusChoices.RUNNING,
             )
-            .only("id", "pid", "started_at", "machine_id", "process_type", "status")
+            .only("id", "pid", "started_at", "machine_id", "process_type", "status", "env", "cmd")
             .order_by("-started_at")
             .first()
             if machine_id is not None
             else None
         )
-        runner_worker = None
         orchestrator_proc_running = bool(orchestrator_proc and orchestrator_proc.is_running)
-        if not orchestrator_proc_running:
-            try:
-                from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
-
-                supervisor = get_existing_supervisord_process(quiet=True)
-                runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
-            except (OSError, RuntimeError, TimeoutError):
-                runner_worker = None
-
-        runner_worker_running = bool(runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"))
-        runner_worker_pid = runner_worker.get("pid") if runner_worker else None
-        orchestrator_running = orchestrator_proc_running or runner_worker_running
-        orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else runner_worker_pid
+        orchestrator_running = orchestrator_proc_running
+        orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else None
 
         # Get model counts by status
         crawl_status_counts = Crawl.status_counts(
             crawl_scope,
-            (Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.PAUSED),
+            (Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED),
         )
         crawls_queued = crawl_status_counts.get(Crawl.StatusChoices.QUEUED, 0)
         crawls_active = crawl_status_counts.get(Crawl.StatusChoices.STARTED, 0)
@@ -286,7 +225,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
 
         snapshot_status_counts = Snapshot.status_counts(
             snapshot_scope,
-            Snapshot.OPEN_STATES,
+            Snapshot.RUNNABLE_STATES,
         )
         snapshots_queued = snapshot_status_counts.get(Snapshot.StatusChoices.QUEUED, 0)
         snapshots_active = snapshot_status_counts.get(Snapshot.StatusChoices.STARTED, 0)
@@ -296,15 +235,24 @@ def live_progress_view(request, *, authorized_snapshot=None):
             ArchiveResult.StatusChoices.QUEUED,
             ArchiveResult.StatusChoices.STARTED,
         )
-        archiveresult_status_counts = ArchiveResult.status_counts(archiveresult_scope, result_statuses)
+        # Start from open results using the existing status index. Filtering by
+        # plugin first scans historical runs, including all completed results.
+        result_counts = list(
+            archiveresult_scope.filter(status__in=result_statuses).order_by().values("status", "plugin").annotate(count=Count("*")),
+        )
+        archiveresult_status_counts = {
+            status: sum(row["count"] for row in result_counts if row["status"] == status) for status in result_statuses
+        }
         download_scope = archiveresult_scope.filter(
             plugin__in=download_plugin_names,
             snapshot__status__in=Snapshot.RUNNABLE_STATES,
             snapshot__crawl__status__in=Crawl.RUNNABLE_STATES,
         )
-        indexing_scope = archiveresult_scope.filter(plugin__in=indexing_plugin_names)
         download_status_counts = ArchiveResult.status_counts(download_scope, result_statuses)
-        indexing_status_counts = ArchiveResult.status_counts(indexing_scope, result_statuses)
+        indexing_status_counts = {
+            status: sum(row["count"] for row in result_counts if row["status"] == status and row["plugin"] in indexing_plugin_names)
+            for status in result_statuses
+        }
         archiveresults_queued = archiveresult_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
         archiveresults_active = archiveresult_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
 
@@ -340,21 +288,27 @@ def live_progress_view(request, *, authorized_snapshot=None):
             .values(*active_crawl_fields)
             .order_by("-modified_at")[:max_active_crawls],
         )
-        paused_crawls = list(
-            crawl_scope.filter(
-                Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
-                | Q(
-                    status=Crawl.StatusChoices.PAUSED,
-                    snapshot_set__status__in=Snapshot.RUNNABLE_STATES,
-                    snapshot_set__retry_at__lte=now,
-                )
-                | Q(
-                    status=Crawl.StatusChoices.PAUSED,
-                    snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
-                ),
+        # Start with runnable children, not every historical snapshot belonging
+        # to a paused crawl. These uncorrelated subqueries run once per poll.
+        due_paused_snapshots = (
+            Snapshot.objects.filter(
+                status__in=Snapshot.RUNNABLE_STATES,
+                retry_at__lte=now,
             )
+            .order_by()
+            .values("crawl_id")
+        )
+        queued_paused_results = (
+            ArchiveResult.objects.filter(
+                status=ArchiveResult.StatusChoices.QUEUED,
+            )
+            .order_by()
+            .values("snapshot__crawl_id")
+        )
+        paused_crawls = list(
+            crawl_scope.filter(status=Crawl.StatusChoices.PAUSED)
+            .filter(Q(created_at__gte=paused_crawl_cutoff) | Q(id__in=due_paused_snapshots) | Q(id__in=queued_paused_results))
             .values(*active_crawl_fields)
-            .distinct()
             .order_by("-modified_at")[:max_active_crawls],
         )
         queued_crawls = list(
@@ -368,52 +322,29 @@ def live_progress_view(request, *, authorized_snapshot=None):
                 crawl["persona_id"] = str(crawl["persona_id"])
         persona_details_by_id: dict[str, dict[str, str]] = {}
         persona_details_by_name: dict[str, dict[str, str]] = {}
-        persona_objects_by_id = {}
-        persona_objects_by_name = {}
         persona_ids = {crawl["persona_id"] for crawl in active_crawls_list if crawl["persona_id"]}
         persona_names = {"Default"} if any(not crawl["persona_id"] for crawl in active_crawls_list) else set()
         if persona_ids or persona_names:
             from archivebox.personas.models import Persona
 
-            for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name", "config"):
+            for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name"):
                 persona_details = {
                     "name": persona.name,
                     "admin_url": f"/admin/personas/persona/{persona.pk}/change/",
                 }
                 persona_details_by_id[str(persona.id)] = persona_details
                 persona_details_by_name[persona.name] = persona_details
-                persona_objects_by_id[str(persona.id)] = persona
-                persona_objects_by_name[persona.name] = persona
         active_crawl_ids = [crawl["id"] for crawl in active_crawls_list]
-        active_crawl_objects = {}
-        if active_crawl_ids:
-            for crawl_obj in Crawl.objects.filter(id__in=active_crawl_ids).select_related("created_by", "persona"):
-                crawl_obj._runtime_config = request_config
-                active_crawl_objects[str(crawl_obj.id)] = crawl_obj
         snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in active_crawl_ids}
-        cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
-        crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
         queued_snapshot_overflow_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
         active_snapshot_scope = snapshot_scope.filter(crawl_id__in=active_crawl_ids)
         if active_crawl_ids:
-            for row in active_snapshot_scope.values("crawl_id", "status").annotate(count=Count("id")):
-                snapshot_counts_by_crawl.setdefault(str(row["crawl_id"]), {})[row["status"]] = row["count"]
-
-            for row in (
-                active_snapshot_scope.filter(status=Snapshot.StatusChoices.SEALED, downloaded_at__isnull=True)
-                .values("crawl_id")
-                .annotate(count=Count("id"))
-            ):
-                cancelled_snapshot_counts_by_crawl[str(row["crawl_id"])] = row["count"]
-
-            for row in (
-                active_snapshot_scope.filter(
-                    status=Snapshot.StatusChoices.SEALED,
-                )
-                .values("crawl_id")
-                .annotate(size=Sum("output_size"))
-            ):
-                crawl_output_sizes_by_crawl[str(row["crawl_id"])] = int(row["size"] or 0)
+            # The existing (crawl, status, modified_at) index covers these counts.
+            # Byte totals and cancellation details require table reads, so sample
+            # those separately and share cached values between viewer tabs.
+            for row in active_snapshot_scope.order_by().values("crawl_id", "status").annotate(count=Count("*")):
+                crawl_id = str(row["crawl_id"])
+                snapshot_counts_by_crawl[crawl_id][row["status"]] = row["count"]
 
         crawl_process_pids: dict[str, int] = {}
         snapshot_process_pids: dict[str, int] = {}
@@ -632,7 +563,14 @@ def live_progress_view(request, *, authorized_snapshot=None):
             completed_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.SEALED, 0)
             started_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.STARTED, 0)
             pending_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.QUEUED, 0)
-            cancelled_snapshots = cancelled_snapshot_counts_by_crawl.get(crawl_id, 0)
+            from archivebox.progressmonitor.collection import crawl_summary
+
+            totals = (
+                crawl_summary(crawl_id, snapshot_id=scoped_snapshot.pk if scoped_snapshot is not None else None)
+                if completed_snapshots
+                else {"snapshots": 0, "cancelled": 0, "bytes": 0}
+            )
+            cancelled_snapshots = totals["cancelled"] if totals is not None else None
 
             # Count URLs in the crawl (for when snapshots haven't been created yet)
             urls_count = 0
@@ -651,7 +589,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
             crawl_setup_completed = sum(1 for item in crawl_setup_plugins if item.get("status") == "succeeded")
             crawl_setup_failed = sum(1 for item in crawl_setup_plugins if item.get("status") == "failed")
             crawl_setup_pending = sum(1 for item in crawl_setup_plugins if item.get("status") == "queued")
-            crawl_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
+            crawl_screencast_url = screencast_frame_url(crawl_id)
             crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id.replace('-', '')}/change/" if crawl_screencast_url else ""
 
             # Get active snapshots for this crawl (already prefetched)
@@ -684,27 +622,28 @@ def live_progress_view(request, *, authorized_snapshot=None):
                 snapshot_screencast_url = ""
                 snapshot_screencast_link = ""
                 snapshot_fallback_urls: list[str] = []
-                result_by_plugin = {result.plugin: result for result in snapshot_results}
-                title_result = result_by_plugin.get("title")
-                if not snapshot_title and title_result is not None and title_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
+                title_result = get_snapshot_role_result(snapshot_results, "title_source")
+                if not snapshot_title and title_result is not None:
                     snapshot_title = Snapshot._normalize_title_candidate(title_result.output_str, snapshot_url=snapshot["url"])
-                favicon_result = result_by_plugin.get("favicon")
-                if favicon_result is not None and favicon_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
-                    favicon_path = archiveresult_output_path(favicon_result) or "favicon/favicon.ico"
-                    snapshot_favicon_url = snapshot_output_url(snapshot, favicon_path)
-                screenshot_result = result_by_plugin.get("screenshot")
-                if screenshot_result is not None and screenshot_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
+                favicon_result = get_snapshot_role_result(snapshot_results, "list_icon")
+                if favicon_result is not None:
+                    favicon_path = favicon_result.embed_path_db() or get_plugin_default_output_path(favicon_result.plugin)
+                    if favicon_path:
+                        snapshot_favicon_url = snapshot_output_url(snapshot, favicon_path)
+                screenshot_result = get_snapshot_role_result(snapshot_results, "primary_preview")
+                if screenshot_result is not None:
                     snapshot_preview_link = snapshot_view_url(snapshot)
-                    screenshot_path = archiveresult_output_path(screenshot_result) or "screenshot/screenshot.png"
-                    snapshot_preview_url = snapshot_output_url(snapshot, screenshot_path)
-                    snapshot_preview_link = snapshot_view_url(snapshot, screenshot_path)
+                    screenshot_path = screenshot_result.embed_path_db() or get_plugin_default_output_path(screenshot_result.plugin)
+                    if screenshot_path:
+                        snapshot_preview_url = snapshot_output_url(snapshot, screenshot_path)
+                        snapshot_preview_link = snapshot_view_url(snapshot, screenshot_path)
                     if snapshot_favicon_url:
                         snapshot_fallback_urls.append(snapshot_favicon_url)
                 elif snapshot_favicon_url:
                     snapshot_preview_url = snapshot_favicon_url
 
                 if snapshot["status"] == Snapshot.StatusChoices.STARTED:
-                    snapshot_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
+                    snapshot_screencast_url = screencast_frame_url(crawl_id)
                     snapshot_screencast_link = snapshot_view_url(snapshot) if snapshot_screencast_url else ""
 
                 def plugin_sort_key(ar):
@@ -752,7 +691,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
                         "process_id": str(process.id) if process else None,
                         "admin_url": f"/admin/core/archiveresult/{ar.id}/change/",
                     }
-                    output_path = archiveresult_output_path(ar)
+                    output_path = ar.embed_path_db()
                     if output_path:
                         plugin_payload["output_path"] = output_path
                         plugin_payload["output_url"] = snapshot_view_url(snapshot, output_path)
@@ -852,18 +791,18 @@ def live_progress_view(request, *, authorized_snapshot=None):
             persona_details = persona_details_by_id.get(str(crawl["persona_id"])) if crawl["persona_id"] else None
             persona_name = persona_details["name"] if persona_details else "Default"
             persona_details = persona_details or persona_details_by_name.get(persona_name)
-            crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0)
-            avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0
-            crawl_obj = active_crawl_objects[crawl_id]
-            effective_crawl_config = get_config(crawl=crawl_obj, resolve_plugins=False)
-            max_urls = int(effective_crawl_config.CRAWL_MAX_URLS or 0)
-            crawl_max_size = int(effective_crawl_config.CRAWL_MAX_SIZE or 0)
-            crawl_timeout = int(effective_crawl_config.CRAWL_TIMEOUT or 0)
-            snapshot_max_size = int(effective_crawl_config.SNAPSHOT_MAX_SIZE or 0)
+            crawl_output_size = totals["bytes"] if totals is not None else None
+            avg_snapshot_size = int(totals["bytes"] / max(totals["snapshots"], 1)) if totals is not None else None
+            # These crawl-scope limits are frozen in Crawl.config. Rendering
+            # them must not resolve personas, probe binaries, or reread config.
+            max_urls = int(crawl["config"].get("CRAWL_MAX_URLS", request_config.CRAWL_MAX_URLS) or 0)
+            crawl_max_size = int(crawl["config"].get("CRAWL_MAX_SIZE", request_config.CRAWL_MAX_SIZE) or 0)
+            crawl_timeout = int(crawl["config"].get("CRAWL_TIMEOUT", request_config.CRAWL_TIMEOUT) or 0)
+            snapshot_max_size = int(crawl["config"].get("SNAPSHOT_MAX_SIZE", request_config.SNAPSHOT_MAX_SIZE) or 0)
 
             # Check if retry_at is in the future (would prevent worker from claiming)
             retry_at_future = crawl["retry_at"] > now if crawl["retry_at"] else False
-            is_paused = crawl_obj.is_paused
+            is_paused = crawl["status"] == Crawl.StatusChoices.PAUSED
             seconds_until_retry = (
                 0 if is_paused else int((crawl["retry_at"] - now).total_seconds()) if crawl["retry_at"] and retry_at_future else 0
             )
@@ -903,8 +842,8 @@ def live_progress_view(request, *, authorized_snapshot=None):
                     "max_snapshot_size_display": printable_filesize(snapshot_max_size) if snapshot_max_size else "unlimited",
                     "crawl_output_size": crawl_output_size,
                     "avg_snapshot_size": avg_snapshot_size,
-                    "crawl_output_size_display": printable_filesize(crawl_output_size) if crawl_output_size else "0 B",
-                    "avg_snapshot_size_display": printable_filesize(avg_snapshot_size) if avg_snapshot_size else "0 B",
+                    "crawl_output_size_display": printable_filesize(crawl_output_size or 0) if totals is not None else "…",
+                    "avg_snapshot_size_display": printable_filesize(avg_snapshot_size or 0) if totals is not None else "…",
                     "tags": crawl_tags,
                     "urls_count": urls_count,
                     "total_snapshots": total_snapshots,

archivebox/search/backends.py

@@ -12,8 +12,11 @@ _search_backends_cache: dict | None = None
 
 def search_backend_command_env(config: dict[str, Any] | None = None, **config_kwargs: Any) -> dict[str, str]:
     """Serialize resolved application config for a standalone plugin command."""
+    from abx_dl.models import PluginEnv
+    from abxpkg import BinProvider
+
     config = config or get_config(**config_kwargs)
-    env = os.environ.copy()
+    env: dict[str, str] = {}
     for key, value in config.items():
         key = str(key)
         if value is None:
@@ -24,7 +27,9 @@ def search_backend_command_env(config: dict[str, Any] | None = None, **config_kw
             env[key] = json.dumps(value)
         elif isinstance(value, (str, int, float, os.PathLike)):
             env[key] = str(value)
-    return env
+    # Use the same runtime bootstrap as hooks so the installation's abxpkg
+    # entrypoint remains available even without an activated virtualenv.
+    return BinProvider.build_exec_env(base_env=PluginEnv(**os.environ).to_env(), extra_env=env)
 
 
 def normalize_search_backend_name(backend_name: str | None) -> str:

archivebox/services/archive_result_service.py

@@ -141,6 +141,23 @@ def _save_archiveresult_event_to_db(
     if snapshot is None:
         return
 
+    if event.status == "cancelled":
+        # Stopping a capture tells the user nothing about whether it would have
+        # succeeded. Remove this attempt instead of showing a failure card or
+        # adding it to "retry failed". The next scheduled run creates a new row.
+        results = ArchiveResult.objects.filter(snapshot=snapshot, plugin=event.plugin, hook_name=event.hook_name)
+        if process_started is not None:
+            # A late completion must not erase a newer retry of the same hook.
+            results = results.filter(start_ts=parse_event_datetime(process_started.start_ts))
+        # Ordinary deletion invokes output-directory cleanup. Cancellation is
+        # only withdrawing the DB fact: retain partial files and sibling hooks'
+        # shared output: they may be the only copy we ever capture. A rerun's
+        # hook owns replacing files without deleting the old copy first (see
+        # abx-plugins README, Rules). Retain Process/logs to explain the stop.
+        results._raw_delete(results.db)
+        ArchiveResult.refresh_snapshot_output_sizes([snapshot.id])
+        return
+
     with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.plugin_dir"):
         plugin_dir = (
             Path(process_started.output_dir)
@@ -193,6 +210,12 @@ def _save_archiveresult_event_to_db(
 
     with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.diff_fields"):
         update_fields = []
+        # This runner already loaded the Snapshot/Crawl for the event. Resolve
+        # retention here without extra parent reads so new results do not wait
+        # for the bounded legacy-repair sweep to traverse historical rows.
+        result.snapshot = snapshot
+        if result.set_delete_at_from_config():
+            update_fields.append("delete_at")
         for field, value in defaults.items():
             if result.__dict__[field] != value:
                 setattr(result, field, value)
@@ -224,7 +247,15 @@ def _save_archiveresult_event_to_db(
 
 
 def mark_archiveresult_started(event: ProcessStartedEvent, *, snapshot_id: str, process_id: str) -> None:
-    """Project a running abx-dl hook after its OS process is persisted."""
+    """Project a running abx-dl hook after its OS process is persisted.
+
+    This DB state records process execution; the scheduler separately waits for
+    stdout readiness before advancing past a background hook. Neither boundary
+    proves that an archive was produced. Final status comes from the owning
+    hook's ArchiveResultEvent, including corrections after an abnormal exit.
+    Rows are per snapshot/plugin/hook: a recorder's start hook cannot declare
+    that its separate export hook succeeded.
+    """
     from archivebox.core.models import ArchiveResult, Snapshot
 
     started_at = parse_event_datetime(event.start_ts)

archivebox/services/resource_admission.py

@@ -2,9 +2,14 @@
 
 import math
 import os
+import re
+import stat
+import sys
 import time
 from pathlib import Path
 
+import psutil
+
 
 RESOURCE_RECHECK_SECONDS = 2.0
 # Scheduling policy, not a minimum-RAM claim: stop increasing extraction
@@ -12,6 +17,65 @@ RESOURCE_RECHECK_SECONDS = 2.0
 # stalls. Require both recent and rolling pressure so transient reclaim does
 # not indefinitely starve otherwise viable work on small hosts.
 MEMORY_FULL_STALL_PERCENT = 10.0
+ZRAM_DEVICE_NAME = re.compile(r"zram\d+")
+
+
+def zram_swap_identifiers(sys_block: Path = Path("/sys/block")) -> set[str]:
+    """Return zram block names and major:minor IDs known to sysfs."""
+    identifiers = set()
+    for device in sys_block.glob("zram*"):
+        if ZRAM_DEVICE_NAME.fullmatch(device.name):
+            identifiers.add(device.name)
+            try:
+                identifiers.add((device / "dev").read_text().strip())
+            except OSError:
+                pass
+    return identifiers
+
+
+def swap_source_identifiers(filename: str) -> set[str]:
+    """Identify a swap source by its visible name and, for block devices, ID."""
+    identifiers = {Path(filename).name}
+    try:
+        source = os.stat(filename)
+        if stat.S_ISBLK(source.st_mode):
+            identifiers.add(f"{os.major(source.st_rdev)}:{os.minor(source.st_rdev)}")
+    except OSError:
+        pass
+    return identifiers
+
+
+def disk_backed_swap_free_bytes(swaps: str, zram_identifiers: set[str]) -> int:
+    """Sum free active swap except zram, whose capacity is already backed by RAM."""
+    lines = swaps.splitlines()
+    free_kib = 0
+    for line in lines[1:]:  # /proc/swaps starts with a column header
+        fields = line.split()
+        if len(fields) < 5:
+            continue
+        filename, _, size_kib, used_kib, _ = fields[:5]
+        source_ids = swap_source_identifiers(filename)
+        if source_ids & zram_identifiers or any(ZRAM_DEVICE_NAME.fullmatch(name) for name in source_ids):
+            continue
+        try:
+            free_kib += max(0, int(size_kib) - int(used_kib))
+        except ValueError:
+            continue
+    return free_kib * 1024
+
+
+def host_swap_free_bytes(meminfo_swap_free: int) -> int:
+    """Use only independently backed swap as extra capacity on Linux."""
+    if not sys.platform.startswith("linux"):
+        return psutil.swap_memory().free
+    try:
+        swaps = Path("/proc/swaps").read_text()
+    except OSError:
+        # If Linux swap sources cannot be classified, fail closed rather than
+        # treating potentially RAM-backed zram capacity as extra headroom.
+        return 0
+    disk_free = disk_backed_swap_free_bytes(swaps, zram_swap_identifiers())
+    return min(max(0, meminfo_swap_free), disk_free)
 
 
 class ResourceAdmission:
@@ -44,10 +108,85 @@ class ResourceAdmission:
                     cpus = min(cpus, max(1, math.ceil(int(quota) / int(period))))
             except (OSError, ValueError, ZeroDivisionError):
                 continue
-        # CPU quota limits parallel admission, not forward progress. A host
-        # with a fractional CPU must still be able to run one snapshot.
+        # CPU quota bounds parallel work; measured memory admission is done
+        # only after the first complete snapshot establishes its actual cost.
         return max(1, min(configured, cpus))
 
+    def additional_snapshot_slots(self, configured: int, active: int, observed_cost: int | None) -> int:
+        if not observed_cost:
+            # Stage one complete real capture before spending memory on parallel
+            # hook trees. A zero-cost/failed probe cannot justify fanout.
+            return 1 if active == 0 and configured > 0 else 0
+        headroom = self.memory_headroom()
+        return self.slots_for_headroom(configured, active, observed_cost, headroom[1] if headroom else 0)
+
+    @staticmethod
+    def slots_for_headroom(configured: int, active: int, observed_cost: int, available: int) -> int:
+        slots = max(0, configured - active)
+        if observed_cost > 0:
+            # Active captures can still grow to the measured peak. Reserve for
+            # that growth before promising the remaining headroom to new work.
+            unpromised = max(0, available - active * observed_cost)
+            slots = min(slots, unpromised // observed_cost)
+        return max(1, slots) if active == 0 and configured else slots
+
+    def memory_headroom(self) -> tuple[int, int] | None:
+        """Return workload RAM+swap usage and effective available headroom."""
+        try:
+            meminfo = {
+                name: int(value.split()[0]) * 1024
+                for name, value in (line.split(":", 1) for line in Path("/proc/meminfo").read_text().splitlines() if ":" in line)
+            }
+            host_swap_free = host_swap_free_bytes(meminfo.get("SwapFree", 0))
+            host_available = meminfo["MemAvailable"] + host_swap_free
+        except (OSError, ValueError, KeyError):
+            host_swap_free = 0 if sys.platform.startswith("linux") else psutil.swap_memory().free
+            host_available = psutil.virtual_memory().available + host_swap_free
+
+        available = host_available
+        used_bytes = None
+        for path in self.cgroup_paths():
+            try:
+                memory_max = (path / "memory.max").read_text().strip()
+                if memory_max == "max":
+                    continue
+                current = int((path / "memory.current").read_text())
+                swap_max = (path / "memory.swap.max").read_text().strip()
+                swap_current = int((path / "memory.swap.current").read_text())
+                swap_room = host_swap_free if swap_max == "max" else min(host_swap_free, max(0, int(swap_max) - swap_current))
+                available = min(available, max(0, int(memory_max) - current) + swap_room)
+                if used_bytes is None:
+                    used_bytes = current + swap_current
+            except (OSError, ValueError, KeyError):
+                continue
+        if used_bytes is None:
+            # Bare-metal and non-Linux installs have no finite cgroup. Count
+            # the real runner process tree rather than the entire host.
+            try:
+                process = psutil.Process()
+                processes = [process, *process.children(recursive=True)]
+            except (psutil.Error, OSError):
+                return None
+            used_bytes = 0
+            for child in processes:
+                try:
+                    if sys.platform.startswith("linux"):
+                        try:
+                            memory = child.memory_full_info()
+                            # Chrome's processes share substantial resident
+                            # memory. RSS charges those pages once per process
+                            # and can falsely serialize captures on an idle host.
+                            used_bytes += memory.pss + memory.swap
+                            continue
+                        except psutil.AccessDenied:
+                            # Restricted /proc access still gets conservative
+                            # accounting through the available RSS measurement.
+                            pass
+                    used_bytes += child.memory_info().rss
+                except (psutil.Error, OSError):
+                    continue
+        return used_bytes, available
+
     def observe_memory_stalls(self, source: str, total: int, avg10: float, sampled_at: float) -> bool:
         previous = self._memory_stalls.get(source)
         self._memory_stalls[source] = (total, sampled_at)

archivebox/services/runner.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 import asyncio
 import contextvars
+import json
+import logging
 import os
 import signal
 import shutil
@@ -10,9 +12,10 @@ import threading
 import time
 from contextlib import nullcontext
 from datetime import timedelta
+from itertools import batched
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Any
+from typing import Any, Literal
 
 from asgiref.sync import sync_to_async
 from django.core.exceptions import ValidationError
@@ -26,6 +29,7 @@ from abx_dl.events import (
     CrawlCleanupEvent,
     CrawlCompletedEvent,
     CrawlEvent,
+    CrawlPauseEvent,
     CrawlSetupEvent,
     CrawlStartEvent,
     InstallEvent,
@@ -36,7 +40,6 @@ from abx_dl.events import (
     SnapshotEvent,
     slow_warning_timeout,
 )
-from abx_dl.limits import CrawlLimitState
 from abx_dl.catalog import PluginCatalog
 from abx_dl.config import GlobalConfig, RuntimeConfig
 from abx_dl.models import Snapshot as AbxSnapshot
@@ -63,6 +66,7 @@ from archivebox.services.resource_admission import RESOURCE_RECHECK_SECONDS, res
 from archivebox.config.common import (
     ArchiveBoxBaseConfig,
     normalize_runtime_config,
+    _is_cookie_file_config_key,
     _plugin_enabled_config_keys,
 )
 from archivebox.misc.db import run_db_analyze_batch
@@ -198,7 +202,12 @@ class CrawlRunner:
         self.crawl = crawl
         self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0)
         self.catalog = get_plugin_catalog()
-        HookProcessService(self.bus, emit_jsonl=False, interactive_tty=interactive_interrupts)
+        self.process_service = HookProcessService(
+            self.bus,
+            emit_jsonl=False,
+            interactive_tty=interactive_interrupts,
+            interrupted_hook_prompt=self._prompt_in_foreground if interactive_interrupts else None,
+        )
         register_sonic_daemon_event_handler(self.bus)
         PersistedProcessService(self.bus)
         ArchiveBoxBinaryService(self.bus)
@@ -209,11 +218,15 @@ class CrawlRunner:
         self.process_discovered_snapshots_inline = process_discovered_snapshots_inline
         self.show_progress = show_progress
         self.interactive_interrupts = interactive_interrupts
+        self._interrupt_choice: asyncio.Future[Literal["abort", "retry", "skip"]] | None = None
+        self._user_aborted = False
+        self.bus.on(CrawlAbortEvent, self.on_CrawlAbortEvent)
         self.config_overrides = dict(config_overrides or {})
 
         self.snapshot_service = SnapshotService(
             self.bus,
             crawl_id=str(crawl.id),
+            interrupted=self.execution_interrupted,
         )
         HookArchiveResultService(self.bus, emit_jsonl=False)
         ArchiveResultService(self.bus)
@@ -223,6 +236,10 @@ class CrawlRunner:
         self.snapshot_tasks: dict[str, asyncio.Task[None]] = {}
         self.snapshot_semaphore = asyncio.Semaphore(1)
         self.max_concurrent_snapshots = 1
+        self._warmup_snapshot_id: str | None = None
+        self._memory_baseline: int | None = None
+        self._memory_peak_cost = 0
+        self._observed_snapshot_cost: int | None = None
         self._resource_deferred = False
         self._initial_snapshot_admitted = False
         self.persona = None
@@ -231,9 +248,11 @@ class CrawlRunner:
         self.primary_url = ""
         self.crawl_output_dir = ""
         self._live_stream = None
+        self._live_ui: LiveBusUI | None = None
         self.root_crawl_event_id: str | None = None
         self.root_crawl_start_event_id: str | None = None
         self._run_task: asyncio.Task[None] | None = None
+        self._run_started = False
         self._skip_wait_until_idle = False
         # This is intentionally a synchronous OS-signal side channel, not bus
         # state. During SIGINT/SIGTERM/SIGHUP, asyncio.run() may already be
@@ -242,21 +261,119 @@ class CrawlRunner:
         self._signal_abort_requested = False
         self._last_lease_heartbeat_at = 0.0
 
-    def _request_abort_from_signal(self, _sig: signal.Signals) -> None:
-        if os.environ.get("ARCHIVEBOX_RUNNER_DAEMON") == "1":
-            # The daemon runner is owned by supervisord, not by the interactive
-            # CLI foreground flow. A direct signal to this child should be short
-            # and unambiguous: exit non-zero immediately so supervisord restarts
-            # the runner, while the parent server and supervisord stay alive.
-            os._exit(128 + int(_sig))
-        already_requested = self._signal_abort_requested
+    def execution_interrupted(self) -> bool:
+        """Owner loss preserves work; explicit user abort keeps its sealing policy."""
+        return self._signal_abort_requested and not self._user_aborted
+
+    async def on_CrawlAbortEvent(self, event: CrawlAbortEvent) -> None:
+        self._signal_abort_requested = True
+        # An abort can arrive while the prompt transport is being established,
+        # before another SIGINT sees _interrupt_choice. Release that waiter too;
+        # shutdown must never depend on a subsequent keypress or IPC response.
+        if self._interrupt_choice is not None and not self._interrupt_choice.done():
+            self._interrupt_choice.set_result("abort")
+        if event.user_initiated:
+            self._user_aborted = True
+            # Exit 130 alone cannot distinguish a user's abort from a worker
+            # displaced by takeover. Tell the owning add command explicitly
+            # so it exits instead of reclaiming and restarting this crawl.
+            if os.environ.get("SUPERVISOR_PROCESS_NAME"):
+                sys.__stderr__.write("ARCHIVEBOX_CRAWL_ABORTED\n")
+                sys.__stderr__.flush()
+
+    def _interrupt_hook(self) -> None:
+        if self._user_aborted:
+            # An explicit abort is already draining hook groups with normal
+            # grace periods. A further Ctrl+C requests immediate force exit.
+            # This service owns only this runner's hook groups and children;
+            # a server's supervisor or another foreground command is outside
+            # the boundary. Tell a supervising add parent this was user intent
+            # before bypassing the worker's normal finalization path.
+            if os.environ.get("SUPERVISOR_PROCESS_NAME"):
+                os.write(sys.__stderr__.fileno(), b"ARCHIVEBOX_CRAWL_ABORTED\n")
+            os.write(sys.__stderr__.fileno(), b"\n[!] Forcing aborted crawl to exit now.\n")
+            self.process_service.force_kill_owned_processes()
+            os._exit(130)
+        if self._run_task is None:
+            # The crawl is already draining/finalizing; there is no live bus on
+            # which to reopen a prompt. Remember the user's abort and finish the
+            # cleanup that is already in progress. In particular do not hand a
+            # second signal to Python's default handler in this teardown window.
+            self._user_aborted = True
+            if os.environ.get("SUPERVISOR_PROCESS_NAME"):
+                os.write(sys.__stderr__.fileno(), b"ARCHIVEBOX_CRAWL_ABORTED\n")
+            stream = self._live_stream if self._live_stream is not None and not self._live_stream.closed else sys.__stderr__
+            os.write(stream.fileno(), b"\nAborting crawl - finishing shutdown... Ctrl+C again to force exit.\n")
+            return
+        if self._interrupt_choice is not None:
+            self._choose_interrupt_action("abort")
+            return
+
+        async def dispatch() -> None:
+            # ArchiveBox transports terminal intent; abx-dl alone decides pause
+            # versus abort. No local counter may compete with its controller.
+            await self.bus.emit(CrawlPauseEvent()).now()
+
+        asyncio.get_running_loop().create_task(dispatch())
+
+    async def _prompt_in_foreground(self, hook_name: str) -> Literal["abort", "retry", "skip"]:
+        rendered = self._live_ui.show_interrupt_prompt(hook_name) if self._live_ui is not None else False
+        if sys.stdin.isatty():
+            # Direct run is itself the outer terminal owner. Keep its asyncio
+            # loop alive so background completions render above the prompt. No
+            # supervisor parent exists here to consume an IPC prompt request.
+            choice = (
+                await HookProcessService.read_interrupt_choice(
+                    hook_name,
+                    render=not rendered,
+                    is_active=lambda: not self._signal_abort_requested,
+                    on_abort=lambda: setattr(self, "_user_aborted", True),
+                )
+                or "abort"
+            )
+            if choice == "abort":
+                # Also cover an inactive prompt that defaulted to abort. A
+                # typed answer is recorded at the keypress by on_abort above.
+                self._user_aborted = True
+            return choice
+        # A supervised worker has log pipes instead of terminal input. Its add
+        # parent owns the same shared reader and returns the answer via signals.
+        self._interrupt_choice = asyncio.get_running_loop().create_future()
+        sys.__stderr__.write("ARCHIVEBOX_HOOK_INTERRUPTED " + json.dumps({"hook_name": hook_name, "rendered": rendered}) + "\n")
+        sys.__stderr__.flush()
+        try:
+            return await self._interrupt_choice
+        finally:
+            self._interrupt_choice = None
+
+    def _choose_interrupt_action(self, choice: Literal["abort", "retry", "skip"]) -> None:
+        if self._interrupt_choice is not None and not self._interrupt_choice.done():
+            if choice == "abort":
+                self._user_aborted = True
+            self._interrupt_choice.set_result(choice)
+
+    def _request_abort_from_signal(self, sig: signal.Signals) -> None:
+        # Daemons also need cooperative cleanup. os._exit used to leave hook
+        # process groups orphaned and Process rows running forever. A daemon's
+        # nonzero exit still lets supervisord restart it, AFTER finally blocks
+        # stop descendants and finalize the ledger. SIGINT uses the same shared
+        # controller as other runners, with interactive_tty=False (whole abort).
+        # Takeover/parent loss stops this execution owner, not the user's crawl
+        # intent. Do not set _user_aborted or emit ARCHIVEBOX_CRAWL_ABORTED here:
+        # the outer add may need to wait for the new owner and reclaim afterward.
+        # Cancellation intentionally bypasses the interactive pause gate; a
+        # waiting prompt must never keep a displaced worker or its hooks alive.
         self._signal_abort_requested = True
         self._skip_wait_until_idle = True
+        if not self._run_started and os.environ.get("ARCHIVEBOX_RUNNER_DAEMON") == "1":
+            # A signal before run() starts has no async task or hooks to drain.
+            # It must still be an unexpected supervisor exit, not a clean idle exit.
+            raise SystemExit(128 + sig.value)
         # The foreground signal handler runs while the event loop may be in the
         # middle of shutdown. Flip cheap in-memory flags here and let normal
-        # finally blocks do cleanup; only cancel the runner task immediately for
-        # non-interactive commands or for a second interrupt escalation.
-        if (not self.interactive_interrupts or already_requested) and self._run_task is not None and not self._run_task.done():
+        # finally blocks do cleanup. Interactive SIGINT uses _interrupt_hook;
+        # actual shutdown signals must cancel promptly in either mode.
+        if self._run_task is not None and not self._run_task.done():
             self._run_task.cancel()
 
     async def crawl_is_cancelled(self) -> bool:
@@ -284,7 +401,7 @@ class CrawlRunner:
             await asyncio.sleep(poll_interval)
             if not await self.crawl_is_cancelled():
                 continue
-            abort_event = parent_event.emit(CrawlAbortEvent())
+            abort_event = parent_event.emit(CrawlAbortEvent(user_initiated=False))
             await _run_event_now(abort_event, abort_event.event_timeout)
             return
 
@@ -299,25 +416,36 @@ class CrawlRunner:
         root_snapshot_id: str | None = None
         bus_destroyed = False
         run_state_loaded = False
-        try:
-            first_signal_message = (
-                "\n[🛑] Got {signal_name}, aborting the active hook...\n"
-                if self.interactive_interrupts
-                else "\n[🛑] Got {signal_name}, stopping gracefully...\n"
-            )
-            self._run_task = asyncio.current_task()
-            # Do not raise KeyboardInterrupt directly from an OS signal while
-            # the asyncio loop is active. Python can inject it into whichever
-            # task is currently running, which produces noisy "Task exception
-            # was never retrieved" logs from unrelated abxbus housekeeping
-            # tasks. _request_abort_from_signal() cancels the runner task
-            # cooperatively instead; repeated signals still hard-exit in the
-            # shared foreground signal handler.
-            with foreground_shutdown_signals(
-                first_signal_message=first_signal_message,
-                on_signal=self._request_abort_from_signal,
-                raise_on_first_signal=False,
-            ):
+        self._run_task = asyncio.current_task()
+        self._run_started = True
+        # Do not raise KeyboardInterrupt directly from an OS signal while
+        # the asyncio loop is active. Python can inject it into whichever
+        # task is currently running, which produces noisy "Task exception
+        # was never retrieved" logs from unrelated abxbus housekeeping
+        # tasks. _request_abort_from_signal() cancels the runner task
+        # cooperatively instead; repeated signals still hard-exit in the
+        # shared foreground signal handler.
+        # Keep signal ownership through bus destruction AND durable finalization.
+        # Restoring the outer/default handler before those awaits leaves a real
+        # race: a rapid second Ctrl+C can kill Python during teardown, producing
+        # supervisor exit -1 / shell 255 instead of an acknowledged user abort.
+        with foreground_shutdown_signals(
+            first_signal_message="\n[🛑] Got {signal_name}, stopping crawl worker...\n",
+            on_signal=self._request_abort_from_signal,
+            interrupt_handlers={
+                signal.SIGINT: self._interrupt_hook,
+                **(
+                    {
+                        signal.SIGUSR1: lambda: self._choose_interrupt_action("skip"),
+                        signal.SIGUSR2: lambda: self._choose_interrupt_action("retry"),
+                    }
+                    if self.interactive_interrupts
+                    else {}
+                ),
+            },
+            raise_on_first_signal=False,
+        ):
+            try:
                 if await sync_to_async(defer_crawl_for_resources, thread_sensitive=True)(self.crawl):
                     return
                 # Startup and its first snapshot are one admission. Browser
@@ -330,6 +458,7 @@ class CrawlRunner:
                 self.max_concurrent_snapshots = max_concurrent_snapshots
                 self.snapshot_semaphore = asyncio.Semaphore(max_concurrent_snapshots)
                 live_ui = self._create_live_ui()
+                self._live_ui = live_ui
                 with live_ui if live_ui is not None else nullcontext():
                     try:
                         if snapshot_ids:
@@ -339,26 +468,30 @@ class CrawlRunner:
                         self._run_task = None
                         await self.stop_snapshot_tasks()
                         try:
+                            await self.process_service.stop_background_hooks()
                             await self.bus.wait_until_idle(timeout=1.0 if self._skip_wait_until_idle else 30.0)
                         except TimeoutError:
                             pass
                         finally:
                             await self.bus.destroy(clear=False)
                             bus_destroyed = True
-        finally:
-            if not bus_destroyed:
-                self._run_task = None
-                await self.stop_snapshot_tasks()
-                await self.bus.destroy(clear=False)
-            if self._live_stream is not None:
-                try:
-                    self._live_stream.close()
-                except Exception:
-                    pass
-                self._live_stream = None
-            if run_state_loaded:
-                await sync_to_async(project_abxpkg_derived_cache_to_db, thread_sensitive=True)(self.base_config.get("ABXPKG_LIB_DIR"))
-                await sync_to_async(self.finalize_run_state, thread_sensitive=True)()
+            finally:
+                if not bus_destroyed:
+                    self._run_task = None
+                    await self.stop_snapshot_tasks()
+                    await self.process_service.stop_background_hooks()
+                    await self.bus.destroy(clear=False)
+                if self._live_stream is not None:
+                    try:
+                        self._live_stream.close()
+                    except Exception:
+                        pass
+                    self._live_stream = None
+                if run_state_loaded:
+                    await sync_to_async(project_abxpkg_derived_cache_to_db, thread_sensitive=True)(self.base_config.get("ABXPKG_LIB_DIR"))
+                    await sync_to_async(self.finalize_run_state, thread_sensitive=True)()
+            if self._user_aborted:
+                raise KeyboardInterrupt
 
     async def enqueue_snapshot(self, snapshot_id: str, crawl_start_event: CrawlStartEvent | None = None) -> None:
         if self._resource_deferred:
@@ -377,10 +510,15 @@ class CrawlRunner:
         if task is not None and not task.done():
             return
         current_event = crawl_start_event or get_current_event()
+        if not isinstance(current_event, CrawlStartEvent) and in_handler_context():
+            return
+        if self._observed_snapshot_cost is None and self._warmup_snapshot_id is None:
+            observation = resource_admission.memory_headroom()
+            self._memory_baseline = observation[0] if observation is not None else None
+            self._memory_peak_cost = 0
+            self._warmup_snapshot_id = snapshot_id
         if isinstance(current_event, CrawlStartEvent):
             task = asyncio.create_task(self.run_snapshot(snapshot_id, current_event), context=_runner_task_context())
-        elif in_handler_context():
-            return
         else:
             task = asyncio.create_task(self.run_snapshot(snapshot_id), context=_runner_task_context())
         self._initial_snapshot_admitted = False
@@ -409,6 +547,7 @@ class CrawlRunner:
                 if task.done():
                     if self.snapshot_tasks.get(snapshot_id) is task:
                         self.snapshot_tasks.pop(snapshot_id, None)
+                    self._finish_warmup_snapshot(snapshot_id)
                     try:
                         task.result()
                     except asyncio.CancelledError as err:
@@ -433,13 +572,16 @@ class CrawlRunner:
                     return
                 continue
             await self.heartbeat_active_leases()
-            done, _pending = await asyncio.wait(pending_tasks, timeout=10.0, return_when=asyncio.FIRST_COMPLETED)
+            self._observe_snapshot_memory()
+            done, _pending = await asyncio.wait(pending_tasks, timeout=RESOURCE_RECHECK_SECONDS, return_when=asyncio.FIRST_COMPLETED)
+            self._observe_snapshot_memory()
             if not done:
                 continue
             for task in done:
                 for snapshot_id, tracked_task in list(self.snapshot_tasks.items()):
                     if tracked_task is task:
                         self.snapshot_tasks.pop(snapshot_id, None)
+                        self._finish_warmup_snapshot(snapshot_id)
                         break
                 try:
                     task.result()
@@ -457,6 +599,41 @@ class CrawlRunner:
             if not stop_scheduling and enqueue_projected:
                 await self.enqueue_pending_snapshots_from_projection()
 
+    def _observe_snapshot_memory(self) -> None:
+        if self._memory_baseline is None:
+            return
+        observation = resource_admission.memory_headroom()
+        if observation is not None:
+            used, _available = observation
+            # Only attribute this workload's growth to a capture. Other jobs on
+            # the host reduce available headroom, but are not a per-capture cost
+            # to multiply again for every snapshot we want to admit.
+            self._memory_peak_cost = max(
+                self._memory_peak_cost,
+                used - self._memory_baseline,
+            )
+            if self._observed_snapshot_cost is not None:
+                self._observed_snapshot_cost = max(self._observed_snapshot_cost, self._memory_peak_cost)
+
+    def _finish_warmup_snapshot(self, snapshot_id: str) -> None:
+        if snapshot_id != self._warmup_snapshot_id or self._observed_snapshot_cost is not None:
+            return
+        self._observe_snapshot_memory()
+        if self._memory_peak_cost > 0:
+            self._observed_snapshot_cost = self._memory_peak_cost
+            headroom = resource_admission.memory_headroom()
+            logging.getLogger(__name__).info(
+                "Snapshot admission: concurrency limit=%s, observed memory growth=%s bytes, available=%s bytes",
+                self.max_concurrent_snapshots,
+                self._observed_snapshot_cost,
+                headroom[1] if headroom else None,
+            )
+        else:
+            # No observed cost cannot justify parallel admission. Probe the
+            # next real snapshot while the queue keeps moving one at a time.
+            self._memory_baseline = None
+        self._warmup_snapshot_id = None
+
     async def heartbeat_active_leases(self) -> None:
         # These are resumable work-item leases, not orchestrator-election
         # heartbeats. Each update is a short autocommit statement; network and
@@ -524,7 +701,11 @@ class CrawlRunner:
         self.max_concurrent_snapshots = resource_admission.snapshot_limit(int(config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]))
 
         active_snapshot_ids = [snapshot_id for snapshot_id, task in self.snapshot_tasks.items() if not task.done()]
-        available_slots = max(0, self.max_concurrent_snapshots - len(active_snapshot_ids))
+        available_slots = resource_admission.additional_snapshot_slots(
+            self.max_concurrent_snapshots,
+            len(active_snapshot_ids),
+            self._observed_snapshot_cost,
+        )
         if available_slots <= 0:
             return
         pending_snapshot_ids = await sync_to_async(
@@ -548,6 +729,9 @@ class CrawlRunner:
         from archivebox.machine.models import Machine, NetworkInterface, Process
 
         self.primary_url = self.crawl.get_urls_list()[0] if self.crawl.get_urls_list() else ""
+        # Validate saved binary paths at the execution boundary, including
+        # config edited since this process first registered its Machine.
+        machine = Machine._sanitize_config(Machine.current())
         current_iface = NetworkInterface.current(refresh=not self.allow_maintenance_on_inactive_crawl)
         current_process = Process.current()
         if current_process.iface_id != current_iface.id or current_process.machine_id != current_iface.machine_id:
@@ -556,8 +740,16 @@ class CrawlRunner:
             current_process.save(update_fields=["iface", "machine", "modified_at"])
         self.persona = self.crawl.resolve_persona()
         self.base_config = get_config(crawl=self.crawl, overrides=self.config_overrides)
-        self.derived_config = dict(Machine.current().config or {})
+        # The merged user payload owns auth paths and binary selections. The
+        # install phase publishes resolved binaries; stale machine paths must
+        # not override explicit runtime settings before that phase runs.
+        self.derived_config = {
+            key: value
+            for key, value in (machine.config or {}).items()
+            if not _is_cookie_file_config_key(key) and (not key.endswith("_BINARY") or value == self.base_config.get(key))
+        }
         self.crawl_output_dir = str(self.crawl.output_dir)
+        Path(self.crawl_output_dir).mkdir(parents=True, exist_ok=True)
         if self.persona:
             self.base_config.update(
                 self.persona.prepare_runtime_for_crawl(
@@ -697,10 +889,12 @@ class CrawlRunner:
         stderr_is_tty = sys.stderr.isatty()
         interactive_tty = stdout_is_tty or stderr_is_tty
         stream = sys.stderr if stderr_is_tty or not stdout_is_tty else sys.stdout
-        if interactive_tty and os.path.exists("/dev/tty"):
+        progress_tty = os.environ.get("ARCHIVEBOX_PROGRESS_TTY")
+        if progress_tty or (interactive_tty and os.path.exists("/dev/tty")):
             try:
-                self._live_stream = open("/dev/tty", "w", buffering=1, encoding=stream.encoding or "utf-8")
+                self._live_stream = open(progress_tty or "/dev/tty", "w", buffering=1, encoding=stream.encoding or "utf-8")
                 stream = self._live_stream
+                interactive_tty = stream.isatty()
             except OSError:
                 self._live_stream = None
         try:
@@ -764,6 +958,7 @@ class CrawlRunner:
                     },
                 )
         snapshot_output_dir = str(snapshot.output_dir)
+        Path(snapshot_output_dir).mkdir(parents=True, exist_ok=True)
         tags = snapshot.tags_str()
         config = self.base_config.for_crawl_runtime(
             crawl=snapshot.crawl,
@@ -874,7 +1069,8 @@ class CrawlRunner:
             if self.initial_snapshot_ids is not None:
                 remaining_snapshot_ids = iter(snapshot_ids)
                 while True:
-                    batch = list(zip(range(self.max_concurrent_snapshots), remaining_snapshot_ids, strict=False))
+                    slots = resource_admission.additional_snapshot_slots(self.max_concurrent_snapshots, 0, self._observed_snapshot_cost)
+                    batch = list(zip(range(slots), remaining_snapshot_ids, strict=False))
                     if not batch:
                         return
                     for _slot, snapshot_id in batch:
@@ -885,7 +1081,8 @@ class CrawlRunner:
                         await self.enqueue_snapshot(snapshot_id)
                     await self.wait_for_snapshot_tasks(enqueue_projected=False)
             else:
-                for snapshot_id in snapshot_ids[: self.max_concurrent_snapshots]:
+                slots = resource_admission.additional_snapshot_slots(self.max_concurrent_snapshots, 0, self._observed_snapshot_cost)
+                for snapshot_id in snapshot_ids[:slots]:
                     if await self.crawl_is_cancelled():
                         break
                     if await self.crawl_is_paused():
@@ -950,18 +1147,17 @@ class CrawlRunner:
             finally:
                 cancel_watcher.cancel()
                 await asyncio.gather(cancel_watcher, return_exceptions=True)
-            completed_event = event.emit(
-                CrawlCompletedEvent(
-                    url=snapshot["url"],
-                    snapshot_id=snapshot["id"],
-                    output_dir=str(output_dir),
-                ),
-            )
-            # Same signal lifecycle as CrawlCleanupEvent above: completion is a
-            # normal bus event unless the interpreter is already unwinding from
-            # SIGINT/SIGTERM/SIGHUP, where synchronous bus delivery is no
-            # longer a dependable shutdown primitive.
-            if not self._signal_abort_requested:
+            # emit() schedules handlers immediately; guarding only the await
+            # still lets completion projectors run during takeover. An aborted
+            # execution owner must leave durable work for its replacement.
+            if not self.execution_interrupted():
+                completed_event = event.emit(
+                    CrawlCompletedEvent(
+                        url=snapshot["url"],
+                        snapshot_id=snapshot["id"],
+                        output_dir=str(output_dir),
+                    ),
+                )
                 await _run_event_now(completed_event, CrawlCompletedEvent.model_fields["event_timeout"].default)
 
         on_archivebox_CrawlStartEvent.__name__ = "on_archivebox_CrawlStartEvent__run_snapshots"
@@ -1033,7 +1229,9 @@ class CrawlRunner:
             snapshot_selected_plugins = (
                 self.requested_plugins or snapshot["retry_plugins"] or snapshot["selected_plugins"] or self.selected_plugins
             )
-            if snapshot["depth"] > 0 and CrawlLimitState.from_config(snapshot["config"]).get_stop_reason() in (
+            # All remaining URLs share the crawl budget, including explicitly
+            # submitted roots. abx-dl only enforces the current snapshot's size.
+            if await sync_to_async(snapshot["_snapshot"].crawl.limit_stop_reason, thread_sensitive=True)(config=config) in (
                 "crawl_max_size",
                 "crawl_timeout",
             ):
@@ -1058,6 +1256,7 @@ class CrawlRunner:
             user_config_event = MachineEvent(config=config, config_type="user")
             user_config_event.event_parent_id = crawl_start_event.event_id
             await self.bus.emit(user_config_event).now()
+            derived_config_event = None
             if derived_config:
                 derived_config_event = MachineEvent(config=derived_config, config_type="derived")
                 derived_config_event.event_parent_id = crawl_start_event.event_id
@@ -1073,6 +1272,7 @@ class CrawlRunner:
                 snapshot_cleanup_phase_timeout=snapshot_phase_timeout,
                 abort_requested=self.crawl_is_cancelled,
             )
+            projected = False
             try:
                 snapshot_event = SnapshotEvent(
                     url=snapshot["url"],
@@ -1096,6 +1296,9 @@ class CrawlRunner:
                     raise RuntimeError(f"Snapshot {snapshot_id} did not complete")
                 await completed_snapshot.wait(timeout=snapshot_phase_timeout)
                 await completed_snapshot.event_results_list()
+                projected = True
+                if self.execution_interrupted():
+                    return
                 if snapshot["status"] == "sealed":
                     await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
                     return
@@ -1117,6 +1320,16 @@ class CrawlRunner:
                 await sync_to_async(_seal_when_last_snapshot_finished, thread_sensitive=True)()
             finally:
                 snapshot_service.close()
+                # Projection and snapshot cleanup have finished. Historical
+                # capture results now live in the database; only active work
+                # needs its event ancestry available to the downloader.
+                expired = [snapshot_event, user_config_event] if projected else []
+                if projected and derived_config_event is not None:
+                    expired.append(derived_config_event)
+                if projected:
+                    expired.extend(await self.bus.filter("*", child_of=snapshot_event, past=True, future=False))
+                for event in expired:
+                    event.event_ttl = 0
 
     def seal_snapshot_due_to_limit(self, snapshot_id: str) -> None:
         from archivebox.core.models import Snapshot
@@ -1346,7 +1559,9 @@ def _run_due_crawl_locked(crawl, *, lock_seconds: int, interactive_interrupts: b
             # avoids hot-looping on the parent while child work is ready without
             # resurrecting a user cancellation that sealed the crawl after
             # selection.
-            crawl.safe_update(
+            if crawl.status != crawl.StatusChoices.STARTED:
+                return False
+            return crawl.safe_update(
                 {
                     "status": crawl.StatusChoices.STARTED,
                     "retry_at": now + timedelta(seconds=ACTIVE_STATE_LEASE_SECONDS),
@@ -1355,7 +1570,6 @@ def _run_due_crawl_locked(crawl, *, lock_seconds: int, interactive_interrupts: b
                 refresh=False,
                 extra_filter={"status": crawl.StatusChoices.STARTED},
             )
-            return True
         if snapshot_count and not due_active_snapshots:
             if crawl.is_finished():
                 if not crawl.claim_processing_lock(lock_seconds=lock_seconds):
@@ -1468,10 +1682,55 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
             return True
         return run_snapshot_maintenance(str(snapshot.id))
 
+    from archivebox.config.common import get_config
+    from archivebox.plugins.discovery import get_enabled_plugins
+
+    config = get_config(crawl=snapshot.crawl, persona=snapshot.crawl.resolve_persona())
+    configured_plugins = {name.strip().lower() for name in str(config.get("PLUGINS") or "").split(",") if name.strip()}
+    chrome_selected = "chrome" in (configured_plugins or set(get_enabled_plugins(config=config)))
+    if chrome_selected and str(config.get("CHROME_ISOLATION") or "crawl").lower() == "crawl":
+        # The due Snapshot is only the scheduler's admission signal. Running
+        # it as an explicit one-snapshot CrawlRunner tears down CrawlSetup after
+        # every URL, even though crawl isolation owns one browser for the whole
+        # crawl. The crawl lifecycle lock already serializes all of its child
+        # work, so let one runner drain the due siblings and discoveries.
+        if any(
+            process.is_running for process in snapshot.process_set.filter(status="running").order_by("pk").paged_iterator(chunk_size=100)
+        ):
+            snapshot.update_and_requeue(retry_at=timezone.now() + timedelta(seconds=lock_seconds))
+            return True
+        if snapshot.fs_migration_needed:
+            # Crawl isolation bypasses the per-Snapshot path below, so migrate
+            # this admitted Snapshot before the CrawlRunner derives its output
+            # directory. Claim the Snapshot as well as holding the lifecycle
+            # lock: maintenance-only workers use the Snapshot lease directly.
+            if not snapshot.claim_processing_lock(lock_seconds=lock_seconds):
+                return False
+            snapshot.refresh_from_db()
+            claimed_status = snapshot.status
+            claimed_retry_at = snapshot.retry_at
+            if snapshot.fs_migration_needed:
+                snapshot.migrate_filesystem_to_current_version()
+            if not snapshot.safe_update(
+                {"retry_at": timezone.now()},
+                refresh=True,
+                extra_filter={"status": claimed_status, "retry_at": claimed_retry_at},
+            ):
+                return False
+        if not snapshot.crawl.claim_processing_lock(lock_seconds=lock_seconds):
+            return False
+        _runner_console_line(crawl=snapshot.crawl)
+        _run_crawl_locked(
+            str(snapshot.crawl_id),
+            process_discovered_snapshots_inline=True,
+            interactive_interrupts=interactive_interrupts,
+        )
+        return True
+
     if not snapshot.claim_processing_lock(lock_seconds=lock_seconds):
         return False
     snapshot.refresh_from_db()
-    if any(process.is_running for process in snapshot.process_set.filter(status="running").iterator()):
+    if any(process.is_running for process in snapshot.process_set.filter(status="running").order_by("pk").paged_iterator(chunk_size=100)):
         # The Snapshot lease may have expired while an abx-dl hook process is
         # still alive. Preserve the snapshot-level ownership boundary and do
         # not launch a second sequence; ArchiveResult status is irrelevant.
@@ -1645,7 +1904,24 @@ def run_install(*, plugin_names: list[str] | None = None) -> None:
 
 def _first_due_id(queryset, *, newest_first: bool = False):
     ordering = ("-retry_at", "-created_at") if newest_first else ("retry_at", "created_at")
-    return queryset.order_by(*ordering).values_list("id", flat=True).first()
+    # Queue membership comes from retry_at, not status: almost the entire
+    # archive can be sealed while only a handful of rows need maintenance.
+    # Combining both predicates lets SQLite choose the broad status index and
+    # scan/sort archival history on every idle pass (567ms on Cabbage vs 3ms
+    # through retry_at). First materialize bounded due IDs using the existing
+    # queue index, then apply status/parent filters to those IDs. This also
+    # bounds PostgreSQL reads and keeps every cursor closed before a CAS claim.
+    due_ids = (
+        queryset.model.objects.filter(retry_at__lte=timezone.now())
+        .order_by(*ordering)
+        .values_list("id", flat=True)
+        .paged_iterator(chunk_size=100)
+    )
+    for candidate_ids in batched(due_ids, 100):
+        match = queryset.filter(id__in=candidate_ids).order_by(*ordering).values_list("id", flat=True).first()
+        if match is not None:
+            return match
+    return None
 
 
 def _run_due_crawl_status(
@@ -1670,12 +1946,11 @@ def _run_due_crawl_status(
     due_crawl = Crawl.objects.filter(id=due_crawl_id).first()
     if due_crawl is None:
         return True
-    run_due_crawl(
+    return run_due_crawl(
         due_crawl,
         lock_seconds=lock_seconds,
         interactive_interrupts=interactive_interrupts,
     )
-    return True
 
 
 def _run_due_snapshot_query(
@@ -1704,32 +1979,28 @@ def _run_due_snapshot_id(snapshot_id, *, lock_seconds: int, interactive_interrup
     due_snapshot = Snapshot.objects.filter(id=due_snapshot_id).first()
     if due_snapshot is None:
         return True
-    run_due_snapshot(
+    # A lost/busy claim is not progress. Let the scheduler try other work and
+    # reach its idle wait instead of spinning on the same due child forever.
+    return run_due_snapshot(
         due_snapshot,
         lock_seconds=lock_seconds,
         interactive_interrupts=interactive_interrupts,
         runtime_config=runtime_config,
     )
-    return True
 
 
 def _run_due_binary() -> bool:
     from archivebox.machine.models import Binary
 
-    due_binary_id = (
-        Binary.objects.filter(retry_at__lte=timezone.now())
-        .exclude(status=Binary.StatusChoices.INSTALLED)
-        .order_by("retry_at", "created_at")
-        .values_list("id", flat=True)
-        .first()
+    due_binary_id = _first_due_id(
+        Binary.objects.filter(retry_at__lte=timezone.now()).exclude(status=Binary.StatusChoices.INSTALLED),
     )
     if due_binary_id is None:
         return False
     due_binary = Binary.objects.filter(id=due_binary_id).first()
     if due_binary is None:
         return True
-    run_due_binary(due_binary, lock_seconds=60)
-    return True
+    return run_due_binary(due_binary, lock_seconds=60)
 
 
 def run_pending_crawls(
@@ -1749,14 +2020,36 @@ def run_pending_crawls(
     last_recovery_at = 0.0
     last_retention_at = 0.0
     last_retention_repair_at = 0.0
+    retention_backfill_cursors: dict = {}
+    deletion_thread: threading.Thread | None = None
     last_analyze_at = 0.0
     analyze_queue: list[str] | None = None
     analyze_sweep_started_at = 0.0
     orchestrator_started_at = time.monotonic()
+
+    def delete_in_background() -> None:
+        from django.db import connections
+
+        try:
+            Snapshot.delete_requested(batch_size=100)
+        finally:
+            connections.close_all()
+
     while True:
         raise_if_shutdown_requested()
         now_monotonic = time.monotonic()
         if crawl_id is None and now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
+            if daemon:
+                # At most one batch can be doing filesystem I/O. A stuck mount
+                # must not block capture scheduling or runner shutdown; rows
+                # remain pending until cleanup succeeds, including on restart.
+                if (deletion_thread is None or not deletion_thread.is_alive()) and Snapshot.objects.filter(
+                    status=Snapshot.DELETING_STATE,
+                ).exists():
+                    deletion_thread = threading.Thread(target=delete_in_background, name="snapshot-delete", daemon=True)
+                    deletion_thread.start()
+            else:
+                Snapshot.delete_requested(batch_size=100)
             for model in (ArchiveResult, Snapshot, Crawl, Process):
                 # Keep the tight scheduler loop anchored on indexed delete_at
                 # columns only. Backfilling missing delete_at values has to read
@@ -1924,7 +2217,7 @@ def run_pending_crawls(
                 # delete_at in the plugin-result hot path. Running it here keeps
                 # DELETE_AFTER resolution fresh without making every hook event
                 # load parent Snapshot/Crawl config.
-                model.delete_expired(batch_size=100, backfill_missing=True)
+                model.delete_expired(batch_size=100, backfill_missing=True, backfill_cursors=retention_backfill_cursors)
             last_retention_repair_at = now_monotonic
 
         if daemon:
@@ -1940,12 +2233,10 @@ def run_pending_crawls(
             # snapshot detail page out to ~500ms. Refresh stats at most once per
             # 24hr while the queue is idle, and only after the orchestrator has
             # been alive for at least an hour so short server boots / one-off work
-            # never pay the cost. The sweep is batched one table per idle tick;
-            # individual table ANALYZE statements abort after 2min (progress
-            # handler) and the whole sweep is hard-capped at 5min so a
-            # pathological table cannot wedge maintenance forever. Any failure
-            # inside the maintenance hook is swallowed — orchestrator must never
-            # be taken down by stats refresh.
+            # never pay the cost. Each idle tick samples a bounded number of
+            # index entries for one table; never scan the full history while
+            # holding ANALYZE's write transaction. Optional stats refresh must
+            # not prevent queued captures.
             try:
                 if (
                     analyze_queue is None

archivebox/services/snapshot_service.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 
 import asyncio
+from collections.abc import Callable
 from pathlib import Path
 
 from asgiref.sync import sync_to_async
 from django.utils import timezone
 from abx_dl.events import SnapshotCompletedEvent, SnapshotEvent
-from abx_dl.limits import CrawlLimitState
 from abx_dl.services.base import BaseService
 
 
@@ -34,7 +34,7 @@ def project_discovered_snapshots(snapshot_id: str) -> list:
         crawl_output_dir=crawl.output_dir,
         snapshot_output_dir=snapshot.output_dir,
     )
-    if CrawlLimitState.from_config(config).get_stop_reason() in ("crawl_max_size", "crawl_timeout"):
+    if crawl.limit_stop_reason(config=config) in ("crawl_max_size", "crawl_timeout"):
         return []
     return crawl.create_discovered_snapshots(snapshot, discovered_urls, depth=snapshot.depth + 1)
 
@@ -74,10 +74,11 @@ def finalize_completed_snapshot(
             modified_at=timezone.now(),
         )
 
-    # SnapshotCompletedEvent is abx-dl's authoritative signal that the complete
-    # snapshot hook sequence (including cleanup) finished. ArchiveResult rows
-    # are projections of that work, never prerequisites used to decide whether
-    # the Snapshot may seal.
+    # SnapshotCompletedEvent finishes an attempt, including abort cleanup. An
+    # explicit user abort deliberately seals that point-in-time capture; the
+    # runner's interrupted-owner guard below keeps takeover work resumable.
+    # ArchiveResult rows are projections of that work, never prerequisites used
+    # to decide whether the Snapshot may seal.
     if not was_sealed and snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.retry_at == owned_retry_at:
         snapshot.seal()
         snapshot.refresh_from_db()
@@ -114,20 +115,16 @@ def finalize_completed_snapshot(
 def _crawl_limit_stop_reason(crawl) -> str:
     from archivebox.config.common import get_config
 
-    config_model = get_config(crawl=crawl)
-    config = config_model.for_crawl_runtime(
-        crawl=crawl,
-        persona=crawl.resolve_persona(),
-    )
-    return CrawlLimitState.from_config(config).get_stop_reason()
+    return crawl.limit_stop_reason(config=get_config(crawl=crawl))
 
 
 class SnapshotService(BaseService):
     LISTENS_TO = [SnapshotEvent, SnapshotCompletedEvent]
     EMITS = []
 
-    def __init__(self, bus, *, crawl_id: str):
+    def __init__(self, bus, *, crawl_id: str, interrupted: Callable[[], bool] | None = None):
         self.crawl_id = crawl_id
+        self.interrupted = interrupted
         self._run_ownership: dict[str, tuple[str, object, bool, list[str]]] = {}
         self._ownership_lock = asyncio.Lock()
         super().__init__(bus)
@@ -149,7 +146,7 @@ class SnapshotService(BaseService):
             elif snapshot.status not in (Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED):
                 return
             if snapshot.status == Snapshot.StatusChoices.STARTED:
-                await sync_to_async(snapshot.ensure_crawl_symlink, thread_sensitive=True)()
+                await sync_to_async(snapshot.reconcile_filesystem_links, thread_sensitive=True)()
             retry_plugins = [str(name).strip() for name in (snapshot.config or {}).get("RETRY_PLUGINS", []) if str(name).strip()]
             async with self._ownership_lock:
                 self._run_ownership[str(event.snapshot_id)] = (str(event.event_id), snapshot.retry_at, was_sealed, retry_plugins)
@@ -181,6 +178,15 @@ class SnapshotService(BaseService):
             if ownership is None or ownership[0] != str(event.event_parent_id):
                 return
             self._run_ownership.pop(snapshot_id, None)
+        # abx-dl also emits completion after abort cleanup, even before the
+        # first hook. Explicit user aborts seal a point-in-time capture so later
+        # work cannot silently mix observations taken days apart; reopening it
+        # is a separate user action. Takeover only replaces its execution owner.
+        # Consult the runner's synchronous signal state: an exiting owner cannot
+        # rely on another bus event arriving before this completion projector runs.
+        # Leave its durable lease for the next elected runner's startup recovery.
+        if self.interrupted is not None and self.interrupted():
+            return
         _, owned_retry_at, was_sealed, retry_plugins = ownership
         await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(
             event.snapshot_id,

archivebox/templates/admin/base.html

@@ -1036,7 +1036,6 @@
             }
 
             #result_list tbody td {
-                padding: 12px 16px;
                 color: #334155;
                 vertical-align: middle;
             }
@@ -1246,8 +1245,8 @@
             body.change-list #changelist {
                 border: 0 !important;
             }
-            body.change-list #changelist .actions .button,
-            body.change-list #changelist .actions select {
+            :is(body.change-list, .snapshot-changelist-widget) #changelist .actions .button,
+            :is(body.change-list, .snapshot-changelist-widget) #changelist .actions select {
                 display: inline-flex;
                 align-items: center;
                 box-sizing: border-box;
@@ -1340,8 +1339,8 @@
             }
 
             /* The native select receives the actual click, including in WKWebView. */
-            body.change-list #changelist .actions .action-rearchive-select,
-            body.change-list #changelist .actions .action-permissions-select {
+            :is(body.change-list, .snapshot-changelist-widget) #changelist .actions .action-rearchive-select,
+            :is(body.change-list, .snapshot-changelist-widget) #changelist .actions .action-permissions-select {
                 position: absolute;
                 inset: 0;
                 width: 100%;
@@ -1352,9 +1351,9 @@
                 margin: 0;
             }
 
-            body.change-list #changelist .action-rearchive-wrapper > .button,
-            body.change-list #changelist .action-permissions-wrapper > .button {
-                padding: 8px 16px !important;
+            :is(body.change-list, .snapshot-changelist-widget) #changelist .action-rearchive-wrapper > .button,
+            :is(body.change-list, .snapshot-changelist-widget) #changelist .action-permissions-wrapper > .button {
+                padding: 8px 18px !important;
                 width: max-content;
                 min-width: max-content;
                 flex: 0 0 auto;
@@ -1528,6 +1527,9 @@
                 display: inline-flex;
                 flex-wrap: wrap;
                 align-items: center;
+                min-width: 0;
+                max-width: 100%;
+                box-sizing: border-box;
                 gap: 4px;
                 padding: 2px 4px;
                 background: transparent;
@@ -1541,15 +1543,27 @@
                 flex-wrap: wrap;
                 gap: 4px;
                 align-items: center;
+                min-width: 0;
+                max-width: 100%;
             }
 
             .tag-editor-inline .tag-pill {
+                min-width: 0;
+                max-width: min(160px, 100%);
+                box-sizing: border-box;
                 padding: 2px 6px 2px 8px;
                 font-size: 11px;
                 border-radius: 12px;
             }
 
+            .tag-editor-inline .tag-link {
+                min-width: 0;
+                overflow: hidden;
+                text-overflow: ellipsis;
+            }
+
             .tag-editor-inline .tag-remove-btn {
+                flex-shrink: 0;
                 width: 14px;
                 height: 14px;
                 font-size: 12px;
@@ -2548,6 +2562,7 @@
             }
         </script>
         <script src="{% static 'admin-inline-tags.js' %}"></script>
+        <script src="{% static 'files-icon-piles.js' %}"></script>
         {% include "includes/output_delete_controls.html" %}
     </body>
 </html>