Bump release version to 0.9.72rc105

This reverts commit da1665acb3.

.github/CONTRIBUTING.md

@@ -12,7 +12,7 @@
 - https://github.com/ArchiveBox/ArchiveBox/issues
 - https://github.com/ArchiveBox/ArchiveBox/pulls
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup
 
 ### Development Setup
 

.github/STAGING.md

@@ -33,3 +33,10 @@ screenshots are retained as Actions artifacts. Other failures remain
 failures; do not publish a success deployment record to bypass them. No code
 mounts are allowed. The Cabbage job must pass before its record authorizes a
 stable upload. Queued capture progress is polled; failed tests are not retried.
+
+Browser acceptance follows the default WACZ viewer into the captured document,
+matching its replay base and original URL and checking that its body is visible
+and contains substantial text. An attached viewer iframe can still be
+`about:blank`, so its presence alone cannot prove replay works. Extracted display
+titles and the captured document's `<title>` may legitimately differ; neither
+title equality nor output thumbnails substitute for checking the actual page.

.github/dependabot.yml

@@ -15,7 +15,7 @@ updates:
         patterns:
           - "*"
   - package-ecosystem: "npm"
-    directory: "/"
+    directory: "/etc"
     target-branch: "dev"
     schedule:
       interval: "monthly"

.github/lychee.toml

@@ -0,0 +1,38 @@
+# Documentation destinations only: do not crawl archived URLs, app routes, or image assets.
+# Manual run from the repository root with Lychee 0.24.2:
+# git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > /tmp/archivebox-link-inputs.txt
+# lychee --config .github/lychee.toml --root-dir "$PWD" --files-from /tmp/archivebox-link-inputs.txt
+# Explicit tracked inputs include README symlinks and let errors in one file leave others checkable.
+include_fragments = "anchor-only"
+include_verbatim = true
+max_retries = 0
+max_concurrency = 4
+timeout = 20
+no_progress = true
+hidden = true
+extensions = ["md", "html", "py", "rst", "yml", "yaml", "sh", "toml"]
+exclude_path = ['(^|/)(tests|data|apidocs|node_modules|\.git)(/|$)']
+# Includes override this exclude; this is a documentation-link allowlist.
+exclude = ['.*']
+include = [
+ '^https://github\.com/ArchiveBox/ArchiveBox/blob/[^/]+/.*\.md(?:#|$)',
+ '^https://raw\.githubusercontent\.com/wiki/ArchiveBox/ArchiveBox/',
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox(?:/wiki(?:/|#|$)|/?#)',
+ '^https?://docs\.archivebox\.io(?:/|$)',
+ '^file:.*\.md(?:#|$)',
+ '^file:.*/docs/[^/.#]+(?:#|$)',
+]
+fallback_extensions = ["md"]
+remap = [
+ # Lychee's Markdown slugger drops variation selectors. Check these against rendered HTML.
+
+ '^file:.*/README\.md(#%EF%B8%8F-cli-usage)$ https://github.com/ArchiveBox/ArchiveBox${1}',
+ '^file:.*/docs/Install\.md(#option-a-docker--docker-compose-setup-%EF%B8%8F)$ https://github.com/ArchiveBox/ArchiveBox/wiki/Install${1}',
+ # The wiki accepts this case alias, but raw file paths are case-sensitive.
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/Setting-up-Storage$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/Setting-Up-Storage.md',
+
+ # Bare missing wiki pages silently redirect to Home; raw pages return an actual 404.
+ # Keep fragment URLs on the rendered wiki to catch published heading drift.
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/([^#/?]+)$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/${1}.md',
+]
+preprocess = { command = "bin/lychee_preprocess.py" }

.github/pages/README.md

@@ -65,10 +65,15 @@ Build: `uv run --no-project python .github/pages/site.py render _site --source p
 Check the built site with real Chromium (also run by Pages CI):
 
 ```sh
-uv run --no-config --no-project --with playwright==1.63.0 playwright install chromium
+export ABXPKG_LIB_DIR=/tmp/archivebox-site-browser
+CHROME_SCHEMA="$(uv run python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
+uv run abxpkg env --install --deps-from="$CHROME_SCHEMA:required_binaries"
+export CHROME_BINARY="$ABXPKG_LIB_DIR/env/bin/chromium"
 uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py SITE_OUTPUT --evidence /tmp/site-evidence
 ```
 
+The browser comes from the normal abxpkg provider/cache path; Playwright drives
+that resolved executable instead of installing a second Chromium build.
 Use this workflow's `SITE_OUTPUT` directory. The check covers desktop/mobile
 layout, keyboard dismissal, no-JavaScript links, local resources and visible
 images, and uploads its screenshots as `site-verification` for 14 days.

.github/pages/artifacts.py

@@ -14,7 +14,7 @@ def trusted(run, repo, workflow, branch, successful=True):
     return (
         (run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
         and run["head_branch"] == branch
-        and run["event"] in ("push", "workflow_dispatch")
+        and run["event"] in ("push", "workflow_dispatch", "schedule")
         and run["path"] == f".github/workflows/{workflow}"
         and (not successful or run["conclusion"] == "success")
     )

.github/pages/base/footer-start.html

@@ -10,4 +10,4 @@ Also review the inline navbar/footer in abx-dl/website/index.html and
 -->
 <footer class="abx-footer" aria-label="ArchiveBox ecosystem">
   <div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
-  <div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps &amp; tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS &amp; iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS &amp; iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated
+  <div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps &amp; tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS &amp; iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS &amp; iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated

.github/pages/verify.py

@@ -2,6 +2,7 @@
 
 import argparse
 import json
+import os
 import re
 from functools import partial
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
@@ -75,7 +76,7 @@ def verify(output, evidence):
     origin = f"http://127.0.0.1:{server.server_port}"
     try:
         with sync_playwright() as playwright:
-            browser = playwright.chromium.launch(ignore_default_args=["--hide-scrollbars"])
+            browser = playwright.chromium.launch(executable_path=os.environ.get("CHROME_BINARY"), ignore_default_args=["--hide-scrollbars"])
             for name in config["pages"]:
                 route = name.removesuffix("index.html")
                 page = browser.new_page(reduced_motion="reduce")

.github/scripts/discover_test_matrix.py

@@ -1,37 +1,79 @@
 #!/usr/bin/env python3
-"""Discover every ArchiveBox test file and expose it as one CI job."""
+"""Discover every test once, batching measured short files."""
 
 import json
+import os
 from pathlib import Path
 
 
 def main() -> None:
-    root = Path.cwd().resolve()
+    root = Path(__file__).resolve().parents[2]
     archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
 
     if not archivebox_tests:
         raise SystemExit("No ArchiveBox tests discovered")
 
-    matrix: list[dict[str, object]] = []
+    # These are observed per-file pytest seconds, including fixture/startup
+    # time. Shared job setup is paid once per batch. Unknown/long files, optional
+    # dependencies and hosted-only requirements keep independent jobs.
+    durations = json.loads((root / ".github/test-durations.json").read_text())["seconds"]
+    hosted = {path for path in archivebox_tests if "# ci-runner: hosted" in path.read_text().splitlines()[:5]}
+    requested = os.environ.get("TEST_FILE", "")
+    batches: list[list[str]] = []
+    batch: list[str] = []
+    batch_seconds = 0
     for path in archivebox_tests:
         test_path = path.relative_to(root).as_posix()
+        duration = durations.get(test_path, 60)
+        if duration >= 60 or path in hosted or path.stem == "test_auth_ldap" or test_path == requested:
+            batches.append([test_path])
+            continue
+        if batch and (len(batch) == 8 or batch_seconds + duration > 180):
+            batches.append(batch)
+            batch = []
+            batch_seconds = 0
+        batch.append(test_path)
+        batch_seconds += duration
+    if batch:
+        batches.append(batch)
+
+    matrix: list[dict[str, object]] = []
+    for paths in batches:
         matrix.append(
             {
-                "name": f"main/{path.stem}",
-                "paths": [test_path],
-                "paths_arg": test_path,
-                "extra": "ldap" if test_path.endswith("/test_auth_ldap.py") else "",
-                "count": 1,
+                "name": f"main/{Path(paths[0]).stem}" if len(paths) == 1 else f"batch/{Path(paths[0]).stem}",
+                "paths": paths,
+                "paths_arg": " ".join(paths),
+                "extra": "ldap" if paths == ["archivebox/tests/test_auth_ldap.py"] else "",
+                "count": len(paths),
+                "ugnas": False,
             },
         )
 
+    # Ordinary Linux tests can use either pool. Only genuine hosted-only
+    # requirements need a file header; filenames never enter runner policy.
+    # Keep a bounded NAS share so hosted capacity continues working in parallel.
+    # This is queued work, not a worker count: several waves keep the capped
+    # runners occupied while hosted jobs run in parallel.
+    nas_job_limit = int(os.environ.get("UGNAS_CI_MAX_JOBS", "3"))
+    # Keep long jobs early in the hosted queue. Giving the longest jobs to the
+    # CPU-capped NAS instead created a tail after hosted work had finished; use
+    # its extra slots for a bounded share of shorter work without filename rules.
+    matrix.sort(key=lambda item: sum(durations.get(path, 60) for path in item["paths"]), reverse=True)
+    eligible = [item for item in reversed(matrix) if not any(root / path in hosted for path in item["paths"])]
+    for item in eligible[:nas_job_limit]:
+        item["ugnas"] = True
+
     discovered_paths = [path for entry in matrix for path in entry["paths"]]
     if len(discovered_paths) != len(set(discovered_paths)):
         raise SystemExit("Tests were not discovered exactly once")
     if sorted(discovered_paths) != [path.relative_to(root).as_posix() for path in archivebox_tests]:
         raise SystemExit("Discovered test shard coverage does not match test files")
 
-    print(f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs")
+    print(
+        f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs; "
+        f"{sum(item['ugnas'] for item in matrix)} selected for ugNAS, subject to the availability check",
+    )
     print(json.dumps(matrix, separators=(",", ":")))
 
 

.github/test-durations.json

@@ -0,0 +1,176 @@
+{
+  "source_runs": [
+    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37190995942",
+    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37237793977"
+  ],
+  "measurement": "Unbatched files use successful per-file pytest step wall seconds from run 37237793977, including process and fixture startup, on their actual hosted or ugNAS runner. Already-batched files retain prior JUnit measurements plus 9 seconds startup. Queue and shared job setup are excluded; unknown files remain independent.",
+  "seconds": {
+    "archivebox/tests/test_api_discovery_performance.py": 28,
+    "archivebox/tests/test_api_v1_auth_browser_session.py": 17,
+    "archivebox/tests/test_api_v1_auth_check_api_token.py": 15,
+    "archivebox/tests/test_api_v1_auth_get_api_token.py": 16,
+    "archivebox/tests/test_api_v1_cli_add.py": 52,
+    "archivebox/tests/test_api_v1_cli_remove.py": 33,
+    "archivebox/tests/test_api_v1_cli_schedule.py": 29,
+    "archivebox/tests/test_api_v1_cli_search.py": 15,
+    "archivebox/tests/test_api_v1_cli_update.py": 60,
+    "archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 59,
+    "archivebox/tests/test_api_v1_core_any_id.py": 13,
+    "archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 24,
+    "archivebox/tests/test_api_v1_core_archiveresults.py": 19,
+    "archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 106,
+    "archivebox/tests/test_api_v1_core_snapshots.py": 17,
+    "archivebox/tests/test_api_v1_core_snapshots_rss.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id.py": 15,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 16,
+    "archivebox/tests/test_api_v1_core_tags.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 16,
+    "archivebox/tests/test_api_v1_core_tags_autocomplete.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_create.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_search.py": 17,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 69,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 15,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 15,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 16,
+    "archivebox/tests/test_api_v1_crawls_crawls.py": 16,
+    "archivebox/tests/test_api_v1_machine_binaries.py": 16,
+    "archivebox/tests/test_api_v1_machine_binary_binary_id.py": 16,
+    "archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 16,
+    "archivebox/tests/test_api_v1_machine_machine_current.py": 15,
+    "archivebox/tests/test_api_v1_machine_machine_machine_id.py": 16,
+    "archivebox/tests/test_api_v1_machine_machines.py": 15,
+    "archivebox/tests/test_api_v1_personas_personas.py": 15,
+    "archivebox/tests/test_api_v1_personas_sync.py": 16,
+    "archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 27,
+    "archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 19,
+    "archivebox/tests/test_archive_result_service.py": 49,
+    "archivebox/tests/test_auth_ldap.py": 37,
+    "archivebox/tests/test_binary_service.py": 46,
+    "archivebox/tests/test_cli_add_1.py": 75,
+    "archivebox/tests/test_cli_add_2.py": 98,
+    "archivebox/tests/test_cli_add_3.py": 93,
+    "archivebox/tests/test_cli_add_4.py": 94,
+    "archivebox/tests/test_cli_archiveresult.py": 48,
+    "archivebox/tests/test_cli_background_screenshot.py": 49,
+    "archivebox/tests/test_cli_binary.py": 12,
+    "archivebox/tests/test_cli_config.py": 88,
+    "archivebox/tests/test_cli_crawl.py": 81,
+    "archivebox/tests/test_cli_extract.py": 97,
+    "archivebox/tests/test_cli_extract_input.py": 84,
+    "archivebox/tests/test_cli_help.py": 16,
+    "archivebox/tests/test_cli_init.py": 111,
+    "archivebox/tests/test_cli_install.py": 38,
+    "archivebox/tests/test_cli_list.py": 116,
+    "archivebox/tests/test_cli_machine.py": 19,
+    "archivebox/tests/test_cli_manage.py": 32,
+    "archivebox/tests/test_cli_mcp.py": 30,
+    "archivebox/tests/test_cli_oneshot.py": 15,
+    "archivebox/tests/test_cli_persona.py": 56,
+    "archivebox/tests/test_cli_piping.py": 118,
+    "archivebox/tests/test_cli_pluginmap.py": 12,
+    "archivebox/tests/test_cli_process.py": 12,
+    "archivebox/tests/test_cli_remove.py": 46,
+    "archivebox/tests/test_cli_run_1.py": 93,
+    "archivebox/tests/test_cli_run_2.py": 111,
+    "archivebox/tests/test_cli_run_3.py": 51,
+    "archivebox/tests/test_cli_run_4.py": 78,
+    "archivebox/tests/test_cli_schedule.py": 59,
+    "archivebox/tests/test_cli_search.py": 25,
+    "archivebox/tests/test_cli_server.py": 126,
+    "archivebox/tests/test_cli_shell.py": 17,
+    "archivebox/tests/test_cli_snapshot.py": 55,
+    "archivebox/tests/test_cli_status.py": 46,
+    "archivebox/tests/test_cli_tag.py": 12,
+    "archivebox/tests/test_cli_update.py": 62,
+    "archivebox/tests/test_cli_update_reindex_snapshots.py": 84,
+    "archivebox/tests/test_cli_version.py": 81,
+    "archivebox/tests/test_config_DELETE_AFTER.py": 145,
+    "archivebox/tests/test_config_MAX_limits.py": 25,
+    "archivebox/tests/test_config_ONLY_NEW.py": 15,
+    "archivebox/tests/test_config_SAVE_TITLE.py": 59,
+    "archivebox/tests/test_config_URL_filters.py": 15,
+    "archivebox/tests/test_core_config.py": 11,
+    "archivebox/tests/test_core_forms.py": 14,
+    "archivebox/tests/test_crawl_history.py": 60,
+    "archivebox/tests/test_crawl_pause.py": 11,
+    "archivebox/tests/test_crawl_runner.py": 22,
+    "archivebox/tests/test_crawl_runner_2.py": 27,
+    "archivebox/tests/test_crawl_service.py": 19,
+    "archivebox/tests/test_frozen_crawl_config.py": 26,
+    "archivebox/tests/test_hook_result_lifecycle.py": 48,
+    "archivebox/tests/test_hooks.py": 54,
+    "archivebox/tests/test_issue_1896.py": 115,
+    "archivebox/tests/test_machine_models.py": 26,
+    "archivebox/tests/test_machine_service.py": 77,
+    "archivebox/tests/test_migrations_04_to_09.py": 25,
+    "archivebox/tests/test_migrations_07_to_09.py": 86,
+    "archivebox/tests/test_migrations_08_to_09.py": 85,
+    "archivebox/tests/test_migrations_fresh.py": 55,
+    "archivebox/tests/test_misc_checks.py": 12,
+    "archivebox/tests/test_misc_monkey_patches.py": 12,
+    "archivebox/tests/test_opencode_agent_1.py": 73,
+    "archivebox/tests/test_opencode_agent_2.py": 141,
+    "archivebox/tests/test_opencode_agent_3.py": 61,
+    "archivebox/tests/test_opencode_browser_1.py": 139,
+    "archivebox/tests/test_opencode_browser_2.py": 164,
+    "archivebox/tests/test_opencode_browser_3.py": 125,
+    "archivebox/tests/test_orm_helpers.py": 14,
+    "archivebox/tests/test_permissions.py": 12,
+    "archivebox/tests/test_persona_casing.py": 45,
+    "archivebox/tests/test_persona_runtime.py": 30,
+    "archivebox/tests/test_postgres_backend.py": 79,
+    "archivebox/tests/test_process_runtime_paths.py": 20,
+    "archivebox/tests/test_process_service.py": 40,
+    "archivebox/tests/test_proof_previews.py": 24,
+    "archivebox/tests/test_recursive_crawl_1.py": 361,
+    "archivebox/tests/test_recursive_crawl_2.py": 55,
+    "archivebox/tests/test_recursive_crawl_3.py": 67,
+    "archivebox/tests/test_release_notes.py": 12,
+    "archivebox/tests/test_release_source_fingerprint.py": 13,
+    "archivebox/tests/test_release_workflow.py": 15,
+    "archivebox/tests/test_resource_admission.py": 13,
+    "archivebox/tests/test_runner_deletion.py": 91,
+    "archivebox/tests/test_runner_watch.py": 75,
+    "archivebox/tests/test_search.py": 142,
+    "archivebox/tests/test_serve_static.py": 19,
+    "archivebox/tests/test_server_automatic_url.py": 22,
+    "archivebox/tests/test_server_security_browser.py": 121,
+    "archivebox/tests/test_settings_signal_webhooks.py": 13,
+    "archivebox/tests/test_setup_script.py": 12,
+    "archivebox/tests/test_shutdown_util.py": 13,
+    "archivebox/tests/test_snapshot_filesystem_migration.py": 16,
+    "archivebox/tests/test_snapshot_output_stacks.py": 63,
+    "archivebox/tests/test_snapshot_ref_lookup.py": 15,
+    "archivebox/tests/test_snapshot_service.py": 48,
+    "archivebox/tests/test_snapshot_url_length.py": 16,
+    "archivebox/tests/test_tag_service.py": 15,
+    "archivebox/tests/test_takeover_util_1.py": 60,
+    "archivebox/tests/test_takeover_util_2.py": 62,
+    "archivebox/tests/test_takeover_util_3.py": 76,
+    "archivebox/tests/test_takeover_util_4.py": 84,
+    "archivebox/tests/test_test_harness.py": 12,
+    "archivebox/tests/test_ui_add_view.py": 26,
+    "archivebox/tests/test_ui_add_view_runtime.py": 120,
+    "archivebox/tests/test_ui_admin_apitoken.py": 16,
+    "archivebox/tests/test_ui_admin_archiveresult.py": 18,
+    "archivebox/tests/test_ui_admin_config_widget.py": 13,
+    "archivebox/tests/test_ui_admin_crawl.py": 25,
+    "archivebox/tests/test_ui_admin_links.py": 23,
+    "archivebox/tests/test_ui_admin_machine.py": 18,
+    "archivebox/tests/test_ui_admin_persona.py": 19,
+    "archivebox/tests/test_ui_admin_snapshot.py": 138,
+    "archivebox/tests/test_ui_admin_tag.py": 16,
+    "archivebox/tests/test_ui_admin_user.py": 16,
+    "archivebox/tests/test_ui_config_views.py": 20,
+    "archivebox/tests/test_ui_live_progress.py": 121,
+    "archivebox/tests/test_ui_public_snapshot.py": 72,
+    "archivebox/tests/test_ui_savepagenow.py": 31,
+    "archivebox/tests/test_urls.py": 105,
+    "archivebox/tests/test_util.py": 13,
+    "archivebox/tests/test_version_metadata.py": 18,
+    "archivebox/tests/test_zip_preview.py": 29
+  }
+}

.github/workflows/ci.yml

@@ -18,51 +18,64 @@ concurrency:
   cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
 
 jobs:
+  prepare:
+    permissions:
+      contents: write
+    uses: ./.github/workflows/prepare-release.yml
+    secrets: inherit
+
   lint:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Linters
     uses: ./.github/workflows/lint.yml
     secrets: inherit
 
   install-cli-platform:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Install and CLI platform compatibility
     uses: ./.github/workflows/test.yml
     secrets: inherit
 
   tests:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Discovered test matrix
     uses: ./.github/workflows/test-parallel.yml
     secrets: inherit
 
-  documentation:
-    name: Documentation and root tests
-    uses: ./.github/workflows/docs.yml
-    secrets: inherit
-
   codeql:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: CodeQL
     uses: ./.github/workflows/codeql.yml
     secrets: inherit
 
   python-artifacts:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Tested Python artifacts
     uses: ./.github/workflows/pip.yml
     secrets: inherit
 
   docker-digests:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Tested Docker images
     uses: ./.github/workflows/docker.yml
     with:
-      push_digests: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+      push_digests: ${{ github.event_name == 'push' && (github.ref_name == 'dev' || github.ref_name == 'main') }}
     secrets: inherit
 
   required:
     name: All required CI lanes
-    if: ${{ always() }}
+    if: ${{ always() && needs.prepare.outputs.run_jobs == 'true' }}
     needs:
+      - prepare
       - lint
       - install-cli-platform
       - tests
-      - documentation
       - codeql
       - python-artifacts
       - docker-digests
@@ -73,7 +86,6 @@ jobs:
           LINT_RESULT: ${{ needs.lint.result }}
           INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
           TESTS_RESULT: ${{ needs.tests.result }}
-          DOCUMENTATION_RESULT: ${{ needs.documentation.result }}
           CODEQL_RESULT: ${{ needs.codeql.result }}
           PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
           DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
@@ -82,7 +94,6 @@ jobs:
             "$LINT_RESULT" \
             "$INSTALL_CLI_PLATFORM_RESULT" \
             "$TESTS_RESULT" \
-            "$DOCUMENTATION_RESULT" \
             "$CODEQL_RESULT" \
             "$PYTHON_ARTIFACTS_RESULT" \
             "$DOCKER_DIGESTS_RESULT"

.github/workflows/deploy-publicsite.yml

@@ -52,8 +52,17 @@ jobs:
       # END REPOSITORY-SPECIFIC BUILD
 
       - name: Verify website in Chromium
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/site-browser
         run: |
-          uv run --no-config --no-project --with playwright==1.63.0 playwright install --with-deps chromium
+          # Resolve the same supported browser as captures; Playwright is the
+          # driver only and must not provision a separate browser/runtime.
+          ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
+          CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
+          uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
+          CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
+          export CHROME_BINARY
+          test -x "$CHROME_BINARY"
           uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py "$SITE_OUTPUT" --evidence "$RUNNER_TEMP/site-evidence"
       - name: Save website verification evidence
         if: always()

.github/workflows/docker.yml

@@ -65,7 +65,7 @@ jobs:
         if: github.event_name == 'workflow_dispatch'
         shell: bash
         env:
-          SOURCE_REF: ${{ inputs.source_ref }}
+          SOURCE_REF: ${{ inputs.source_ref || github.sha }}
         run: |
           set -Eeuo pipefail
           [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
@@ -274,28 +274,25 @@ jobs:
         if: inputs.push_digests
         run: echo ${{ steps.docker_build.outputs.digest }}
 
-      - name: Verify OpenCode install with a read-only browser profile
+      - name: Verify optional agent install with a read-only browser profile
         shell: bash
         env:
           IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
         run: |
           set -Eeuo pipefail
           PROFILE_DIR="$(mktemp -d)"
+          # shellcheck disable=SC2016
           "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' \
             --volume "$PROFILE_DIR:/home/archivebox/.config/chromium:ro" \
             "$IMAGE" bash -c '
               set -Eeuo pipefail
               test -w "$XDG_CONFIG_HOME"
               test ! -w "$XDG_CONFIG_HOME/chromium"
-              # Check the shipped executable before the installer can repair
-              # an image that accidentally omitted this opt-in dependency.
-              "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
-              archivebox install opencode
-              "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
+              uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_archivebox_install; verify_archivebox_install()"
             '
 
       - name: Validate exact built image version and commit
-        if: ${{ !inputs.push_digests }}
+        if: inputs.full_tests
         shell: bash
         run: |
           set -Eeuo pipefail
@@ -350,6 +347,7 @@ jobs:
           (( COMPRESSED_SIZE < LIMIT ))
 
       - name: Resolve real Docker CLI test dependencies through abxpkg
+        id: mount_dependencies
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash
         run: |
@@ -371,100 +369,142 @@ jobs:
             test -x "$binary"
           done
 
-      - name: Validate Docker UID and GID behavior on real NFS and SMB mounts
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
+      - name: Diagnose failed mount dependency installation
+        if: failure() && steps.mount_dependencies.conclusion == 'failure'
         shell: bash
         run: |
-          set -Eeuo pipefail
-          NFS_SOURCE="/srv/archivebox-ci-nfs"
-          SMB_SOURCE="/srv/archivebox-ci-smb"
-          NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
-          SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
-          mounted_nfs=0
-          mounted_smb=0
+          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
+          sudo journalctl --since '-10 minutes' --no-pager \
+            -u smbd -u nmbd -u samba-ad-dc -u nfs-server
+          ps -eo pid,ppid,stat,etimes,wchan:32,comm
 
-          cleanup() {
-            local original_status=$? cleanup_status=0
-            trap - EXIT
-            if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
-              cleanup_status=1
-            fi
-            if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
-              cleanup_status=1
-            fi
-            if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
-              cleanup_status=1
-            fi
-            if [[ "$original_status" != "0" ]]; then
-              exit "$original_status"
-            fi
-            exit "$cleanup_status"
-          }
-          trap cleanup EXIT
-
-          "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-          "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
-          "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-          "$SUDO_BINARY" "$BASH_BINARY" -c \
-            "printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
-          "$SUDO_BINARY" "$BASH_BINARY" -c \
-            "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
-
-          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
-          "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
-          "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
-          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
-          "$SYSTEMCTL_BINARY" is-active --quiet smbd.service
-
-          "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
-          mounted_nfs=1
-          "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
-            -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
-          mounted_smb=1
-
-          NFS_TEST_DIR="$NFS_MOUNT" \
-          SMB_TEST_DIR="$SMB_MOUNT" \
-          IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
-            "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
-
-      - name: Validate Docker port compatibility
-        shell: bash
-        env:
-          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
-        run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
-
-      - name: Validate Docker CLI add/list and compose behavior
+      - name: Validate Docker mounts, ports, CLI and compose concurrently
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash
         env:
           ARCHIVEBOX_IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
         run: |
           set -Eeuo pipefail
-          "$MKDIR_BINARY" "${{ github.workspace }}/data"
-          "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
+          # These checks own separate collections, Compose projects and ports.
+          # Subshells keep mount cleanup traps and environment changes isolated.
+          validate_mounts() (
+            set -Eeuo pipefail
+            NFS_SOURCE="/srv/archivebox-ci-nfs"
+            SMB_SOURCE="/srv/archivebox-ci-smb"
+            NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
+            SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
+            mounted_nfs=0
+            mounted_smb=0
 
-          "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
-          "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
+            # Invoked by the subshell EXIT trap.
+            # shellcheck disable=SC2329
+            cleanup() {
+              local original_status=$? cleanup_status=0
+              trap - EXIT
+              if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
+                cleanup_status=1
+              fi
+              if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
+                cleanup_status=1
+              fi
+              if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
+                cleanup_status=1
+              fi
+              if [[ "$original_status" != "0" ]]; then
+                exit "$original_status"
+              fi
+              exit "$cleanup_status"
+            }
+            trap cleanup EXIT
 
-          "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
-          echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add
+            "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
+            "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
+            "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
+            "$SUDO_BINARY" "$BASH_BINARY" -c \
+              "printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
+            "$SUDO_BINARY" "$BASH_BINARY" -c \
+              "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
 
-          archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
-          [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
-          [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
+            "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
+            "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
+            "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
+            "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
+            "$SYSTEMCTL_BINARY" is-active --quiet smbd.service
 
-          "$DOCKER_BINARY" compose run --rm archivebox init
-          "$DOCKER_BINARY" compose up -d --wait
-          "$DOCKER_BINARY" network connect archivebox_default www-nginx
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
-          admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
-          [[ "$index_html" == *ArchiveBox* ]]
-          [[ "$admin_js" == *window.django* ]]
+            "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
+            mounted_nfs=1
+            "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
+              -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
+            mounted_smb=1
 
-          "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
-          [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
-          "$DOCKER_BINARY" compose down
+            NFS_TEST_DIR="$NFS_MOUNT" \
+            SMB_TEST_DIR="$SMB_MOUNT" \
+            IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
+              "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
+          )
+          validate_cli() (
+            set -Eeuo pipefail
+            "$MKDIR_BINARY" "${{ github.workspace }}/data"
+            "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
+
+            "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
+            "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
+
+            "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
+            # The argument form above exercises the complete default pipeline.
+            # Keep a real capture for stdin parsing without repeating every plugin.
+            echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add --plugins=wget,hashes
+
+            archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
+            [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
+            [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
+
+            "$DOCKER_BINARY" compose run --rm archivebox init
+            "$DOCKER_BINARY" compose up -d --wait
+            "$DOCKER_BINARY" network connect archivebox_default www-nginx
+            index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
+            admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
+            [[ "$index_html" == *ArchiveBox* ]]
+            [[ "$admin_js" == *window.django* ]]
+
+            "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
+            index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
+            [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
+            "$DOCKER_BINARY" compose down
+          )
+          validate_ports() (
+            IMAGE="$ARCHIVEBOX_IMAGE" "$BASH_BINARY" bin/validate_docker_ports.sh
+          )
+
+          checks=(mounts ports cli)
+          pids=()
+          validate_mounts > "$RUNNER_TEMP/docker-mounts.log" 2>&1 &
+          pids+=("$!")
+          validate_ports > "$RUNNER_TEMP/docker-ports.log" 2>&1 &
+          pids+=("$!")
+          validate_cli > "$RUNNER_TEMP/docker-cli.log" 2>&1 &
+          pids+=("$!")
+          status=0
+          for index in "${!checks[@]}"; do
+            check="${checks[$index]}"
+            echo "::group::Docker $check validation"
+            if wait "${pids[$index]}"; then
+              echo "PASS: Docker $check validation"
+            else
+              echo "::error::Docker $check validation failed"
+              status=1
+            fi
+            cat "$RUNNER_TEMP/docker-$check.log"
+            echo "::endgroup::"
+          done
+          exit "$status"
+
+      - name: Validate Docker port compatibility
+        if: ${{ !(inputs.full_tests && matrix.platform == 'linux/amd64') }}
+        shell: bash
+        env:
+          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
+        run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
 
       - name: Validate compressed image size
         if: inputs.full_tests && !inputs.push_digests

.github/workflows/docs-links.yml

@@ -0,0 +1,38 @@
+name: Documentation link checks
+
+on:
+  schedule:
+    - cron: '17 10 * * *' # Daily at 10:17 UTC, away from the busy top of the hour.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  PYTHONIOENCODING: utf-8
+
+jobs:
+  links:
+    name: Documentation links and anchors
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+      - name: List tracked documentation and source inputs
+        run: |
+          git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > "$RUNNER_TEMP/link-inputs.txt"
+      - uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
+        with:
+          lycheeVersion: v0.24.2
+          args: >-
+            --config .github/lychee.toml
+            --root-dir "${{ github.workspace }}"
+            --files-from "${{ runner.temp }}/link-inputs.txt"
+          fail: true
+          failIfEmpty: true

.github/workflows/docs.yml

@@ -1,7 +1,17 @@
 name: Documentation tests
 
 on:
-  workflow_call:
+  schedule:
+    - cron: '7 5,17 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
 
 env:
   PYTHONIOENCODING: utf-8
@@ -9,7 +19,18 @@ env:
   SHOW_PROGRESS: "False"
 
 jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
   discover:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
     name: Discover documentation examples
     runs-on: ubuntu-24.04
     timeout-minutes: 5
@@ -67,7 +88,7 @@ jobs:
     timeout-minutes: 15
     strategy:
       fail-fast: false
-      max-parallel: 4
+      max-parallel: 8
       matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -158,3 +179,17 @@ jobs:
           "$SUDO_BINARY" \
             --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
             "$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
+      - name: Diagnose failed Linux package installation
+        if: failure() && runner.os == 'Linux'
+        run: |
+          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
+          apt-config dump Acquire::Retries Acquire::http::Timeout Acquire::https::Timeout
+          ps -eo pid,ppid,stat,etimes,wchan:32,comm
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, discover, examples]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled work completed successfully.'

.github/workflows/macos.yml

@@ -0,0 +1,48 @@
+name: macOS compatibility
+
+on:
+  schedule:
+    - cron: '51 2,10,18 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
+  install-cli:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
+    uses: ./.github/workflows/test.yml
+    with:
+      platform: macos
+    secrets: inherit
+
+  python-artifacts:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
+    uses: ./.github/workflows/pip.yml
+    with:
+      platforms: '["macos-15"]'
+    secrets: inherit
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, install-cli, python-artifacts]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled macOS coverage completed successfully.'

.github/workflows/pip.yml

@@ -3,6 +3,11 @@ name: Build Pip package
 on:
   workflow_call:
     inputs:
+      platforms:
+        description: Runner matrix for package compatibility
+        required: false
+        default: '["ubuntu-24.04"]'
+        type: string
       full_tests:
         description: Run the full package compatibility matrix instead of one release smoke check
         required: false
@@ -95,7 +100,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-24.04, macos-15]
+        os: ${{ fromJSON(inputs.platforms) }}
         python: ["3.13", "3.14"]
         artifact: [wheel, sdist]
 

.github/workflows/prepare-release.yml

@@ -1,32 +1,33 @@
-name: Release candidate
+name: Prepare release version
 
 on:
-  push:
-    branches: [dev, main]
+  workflow_call:
+    outputs:
+      run_jobs:
+        description: Whether this revision owns its release version and should run expensive jobs
+        value: ${{ jobs.prepare.outputs.run_jobs }}
 
 permissions:
-  actions: read
   contents: write
-  packages: write
-
-concurrency:
-  group: release-candidate-${{ github.ref_name }}
-  cancel-in-progress: true
 
 jobs:
   prepare:
+    name: Prepare the version before starting expensive jobs
+    permissions:
+      contents: write
     runs-on: ubuntu-24.04
     outputs:
-      action: ${{ steps.version.outputs.action }}
+      run_jobs: ${{ github.event_name != 'push' || steps.channel.outputs.stable == 'true' || steps.version.outputs.action == 'ready' || steps.version.outputs.action == 'reserved' }}
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           fetch-depth: 0
           submodules: true
-          token: ${{ secrets.RELEASE_GH_TOKEN }}
+          token: ${{ github.event_name == 'push' && secrets.RELEASE_GH_TOKEN || github.token }}
 
       - name: Keep synchronized stable branches on the released version
         id: channel
+        if: github.event_name == 'push'
         shell: bash
         run: |
           set -Eeuo pipefail
@@ -45,41 +46,10 @@ jobs:
           fi
 
       - id: version
-        if: steps.channel.outputs.stable != 'true'
+        if: github.event_name == 'push' && steps.channel.outputs.stable != 'true'
         uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
         with:
           package: archivebox
           branch: ${{ github.ref_name }}
           tag-prefix: v
           version-scheme: ${{ github.ref_name == 'main' && 'patch' || 'rc' }}
-
-  python-artifacts:
-    needs: prepare
-    if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
-    uses: ./.github/workflows/pip.yml
-    with:
-      full_tests: false
-    secrets: inherit
-
-  docker-digests:
-    needs: prepare
-    if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
-    uses: ./.github/workflows/docker.yml
-    with:
-      push_digests: true
-      full_tests: false
-    secrets: inherit
-
-  opencode:
-    needs: prepare
-    if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
-    strategy:
-      fail-fast: false
-      matrix:
-        test_file:
-          - archivebox/tests/test_opencode_agent.py
-          - archivebox/tests/test_opencode_browser.py
-    uses: ./.github/workflows/test-parallel.yml
-    with:
-      test_file: ${{ matrix.test_file }}
-    secrets: inherit

.github/workflows/release.yml

@@ -2,7 +2,7 @@ name: Release
 
 on:
   workflow_run:
-    workflows: [CI, Release candidate]
+    workflows: [CI]
     types: [completed]
     branches: [dev, main]
   workflow_dispatch:
@@ -111,26 +111,14 @@ jobs:
               --jq '[.status, .conclusion, .headSha, .event, .name] | @tsv')"
             [[ "$CI_STATUS" == "$(printf 'completed\tsuccess\t%s\tpush\tCI' "$RELEASE_SHA")" ]]
           else
-            CANDIDATE_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow release-candidate.yml \
-              --commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
-              --json databaseId,status,conclusion \
-              --jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
             CI_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml \
               --commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
               --json databaseId,status,conclusion \
               --jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
-            [[ -n "$CANDIDATE_RUN_ID" && -n "$CI_RUN_ID" ]] || exit 0
-            if [[ "$RELEASE_BRANCH" == dev ]]; then
-              DIGEST_RUN_ID="$CANDIDATE_RUN_ID"
-            else
-              DIGEST_RUN_ID="$CI_RUN_ID"
-            fi
-          fi
-          if [[ "$RELEASE_BRANCH" == dev && "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
-            ARTIFACT_RUN_ID="$CANDIDATE_RUN_ID"
-          else
-            ARTIFACT_RUN_ID="$CI_RUN_ID"
+            [[ -n "$CI_RUN_ID" ]] || exit 0
+            DIGEST_RUN_ID="$CI_RUN_ID"
           fi
+          ARTIFACT_RUN_ID="$CI_RUN_ID"
           {
             echo 'ready=true'
             echo "artifact_run_id=$ARTIFACT_RUN_ID"
@@ -404,7 +392,8 @@ jobs:
 
       - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
         with:
-          install: true
+          # These images were already built and tested; only publish manifests.
+          driver: docker
 
       - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
         with:
@@ -537,17 +526,47 @@ jobs:
           for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
           $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
 
-      - name: Verify published Docker images run
+      - name: Verify published tags contain the exact tested images
+        env:
+          DOCKERHUB_TAGS: ${{ steps.docker_meta.outputs.dockerhub_tags }}
+          GHCR_TAGS: ${{ steps.docker_meta.outputs.ghcr_tags }}
         shell: bash
         run: |
           set -Eeuo pipefail
-          VERSION='${{ needs.python-release.outputs.version }}'
-          SHORT_SHA="${RELEASE_SHA::12}"
-          for image in "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"; do
-            "$DOCKER_BINARY" buildx imagetools inspect "$image" >/dev/null
-            INSTALLED_VERSION="$("$DOCKER_BINARY" run --rm "$image" --version)"
-            test "$INSTALLED_VERSION" = "$VERSION"
-          done
+          # Both architectures already passed CLI/version/capture acceptance by
+          # digest. Verify promotion preserves those exact images and attestations
+          # instead of downloading and running the same image again on one arch.
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
+          EXPECTED="$("$DOCKER_BINARY" buildx imagetools create --dry-run "${REFS[@]}" | "$JQ_BINARY" -ceS '.manifests | sort_by(.digest)')"
+          "$JQ_BINARY" -e 'map(select(.platform.os == "linux") | .platform.architecture) | sort == ["amd64", "arm64"]' <<< "$EXPECTED"
+          while IFS= read -r image; do
+            [[ -n "$image" ]] || continue
+            ACTUAL="$("$DOCKER_BINARY" buildx imagetools inspect --raw "$image" | "$JQ_BINARY" -ceS 'select(.schemaVersion == 2) | .manifests | sort_by(.digest)')"
+            [[ "$ACTUAL" == "$EXPECTED" ]] || { echo "Published tag differs from tested images: $image" >&2; exit 1; }
+            echo "Verified exact tested images: $image"
+          done < <(printf '%s\n' "$DOCKERHUB_TAGS" "$GHCR_TAGS")
+
+      - name: Record published image source
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          # workflow_run can report the default branch's head even when this
+          # run published a different branch/SHA. Preserve the verified target.
+          [[ "$RELEASE_BRANCH" == dev || "$RELEASE_BRANCH" == main ]]
+          [[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          mkdir -p "$RUNNER_TEMP/published-image-source"
+          # shellcheck disable=SC2016
+          "$JQ_BINARY" -n --arg branch "$RELEASE_BRANCH" --arg sha "$RELEASE_SHA" \
+            '{branch: $branch, sha: $sha}' > "$RUNNER_TEMP/published-image-source/source.json"
+
+      - name: Upload published image source
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: published-image-source
+          path: ${{ runner.temp }}/published-image-source/source.json
+          if-no-files-found: error
 
       - name: Update Docker Hub README
         uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4

.github/workflows/screenshots.yml

@@ -1,29 +1,31 @@
 name: Capture publicsite screenshots
 
 on:
-  push:
-    branches:
-      - dev
-    paths:
-      - archivebox/**
-      - bin/collect_ui_screenshots.*
-      - bin/generate_ui_screenshot_gallery.py
-      - bin/take_screenshot.js
-      - docs/Screenshots.md
-      - pyproject.toml
-      - uv.lock
-      - .github/workflows/screenshots.yml
+  schedule:
+    - cron: '37 5,17 * * *'
   workflow_dispatch:
 
 permissions:
   contents: read
+  actions: read
 
 concurrency:
-  group: ${{ github.event_name == 'workflow_dispatch' && format('publicsite-captures-manual-{0}', github.run_id) || 'publicsite-captures' }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: false
 
 jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
   deploy:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout
@@ -69,3 +71,11 @@ jobs:
           path: publicsite/screenshots
           if-no-files-found: error
           retention-days: 90
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, deploy]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled work completed successfully.'

.github/workflows/staging-acceptance.yml

@@ -4,7 +4,6 @@ on:
   workflow_run:
     workflows: [Release]
     types: [completed]
-    branches: [dev]
 
 permissions: {}
 
@@ -14,12 +13,13 @@ concurrency:
 
 jobs:
   published:
-    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'dev'
+    if: github.event.workflow_run.conclusion == 'success'
     runs-on: ubuntu-24.04
     permissions:
       actions: read
     outputs:
-      ready: ${{ steps.image.outputs.ready }}
+      ready: ${{ steps.source.outputs.ready }}
+      sha: ${{ steps.source.outputs.sha }}
     steps:
       - id: image
         env:
@@ -27,8 +27,36 @@ jobs:
           RELEASE_RUN_ID: ${{ github.event.workflow_run.id }}
         run: |
           set -Eeuo pipefail
-          ready=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RELEASE_RUN_ID/jobs?per_page=100" --jq 'any(.jobs[]; .name == "Publish tested multiarch image" and .conclusion == "success")')
-          echo "ready=$ready" >> "$GITHUB_OUTPUT"
+          published=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RELEASE_RUN_ID/jobs?per_page=100" --jq 'any(.jobs[]; .name == "Publish tested multiarch image" and .conclusion == "success")')
+          echo "published=$published" >> "$GITHUB_OUTPUT"
+      - name: Download published image source
+        if: steps.image.outputs.published == 'true'
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: published-image-source
+          path: ${{ runner.temp }}/published-image-source
+          github-token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          run-id: ${{ github.event.workflow_run.id }}
+      - name: Select published dev image
+        id: source
+        if: steps.image.outputs.published == 'true'
+        run: |
+          set -Eeuo pipefail
+          # The triggering run may report the default branch as its head;
+          # this artifact names the image source verified by the publisher.
+          source="$RUNNER_TEMP/published-image-source/source.json"
+          jq -e 'type == "object" and (keys | sort) == ["branch", "sha"]' "$source" >/dev/null
+          branch=$(jq -er '.branch | select(type == "string")' "$source")
+          sha=$(jq -er '.sha | select(type == "string")' "$source")
+          [[ "$branch" == dev || "$branch" == main ]]
+          [[ "$sha" =~ ^[0-9a-f]{40}$ ]]
+          echo "sha=$sha" >> "$GITHUB_OUTPUT"
+          if [[ "$branch" == dev ]]; then
+            echo 'ready=true' >> "$GITHUB_OUTPUT"
+          else
+            echo 'ready=false' >> "$GITHUB_OUTPUT"
+          fi
   acceptance:
     # The privileged key is never exposed to PRs or arbitrary dispatch refs.
     needs: published
@@ -46,7 +74,7 @@ jobs:
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
         with:
-          ref: ${{ github.event.workflow_run.head_sha }}
+          ref: ${{ needs.published.outputs.sha }}
           fetch-depth: 0
           persist-credentials: false
       - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e
@@ -56,17 +84,25 @@ jobs:
         env:
           STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
           STAGING_KNOWN_HOSTS: ${{ secrets.STAGING_KNOWN_HOSTS }}
+          ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/staging-browser
         run: |
           set -Eeuo pipefail
           umask 077
           printf '%s\n' "$STAGING_SSH_KEY" > "$RUNNER_TEMP/staging-key"
           printf '%s\n' "$STAGING_KNOWN_HOSTS" > "$RUNNER_TEMP/staging-known-hosts"
-          uv run --no-project --with playwright==1.63.0 playwright install chromium
+          # Use the candidate's normal provider/cache path, not a second
+          # Playwright-managed browser that bypasses abxpkg validation.
+          ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
+          CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
+          uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
+          CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
+          test -x "$CHROME_BINARY"
+          echo "CHROME_BINARY=$CHROME_BINARY" >> "$GITHUB_ENV"
       - name: Verify actual deployment, full captures and rendered snapshots
         env:
           GH_TOKEN: ${{ github.token }}
           STAGING_TARGET: ${{ matrix.target }}
-          STAGING_SHA: ${{ github.event.workflow_run.head_sha }}
+          STAGING_SHA: ${{ needs.published.outputs.sha }}
           STAGING_KEY_FILE: ${{ runner.temp }}/staging-key
           STAGING_KNOWN_HOSTS_FILE: ${{ runner.temp }}/staging-known-hosts
         run: uv run --no-project --with playwright==1.63.0 python bin/check-staging-deployment.py
@@ -76,6 +112,6 @@ jobs:
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
         if: always()
         with:
-          name: staging-${{ matrix.target }}-${{ github.event.workflow_run.head_sha }}
+          name: staging-${{ matrix.target }}-${{ needs.published.outputs.sha }}
           path: staging-evidence/
           if-no-files-found: error

.github/workflows/test-parallel.yml

@@ -36,14 +36,9 @@ jobs:
       - name: Select available CI capacity
         id: runner
         if: vars.UGNAS_CI_ENABLED == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
-        uses: mikehardy/runner-fallback-action@dc7732f9e532c451b2527b288ba8d58fd4b0a4ca # v1.1.0
+        uses: ArchiveBox/monorepo/.github/actions/select-ci-runner@main
         with:
           github-token: ${{ secrets.RUNNER_STATUS_TOKEN }}
-          organization: ArchiveBox
-          primary-runner: self-hosted,ugnas-linux
-          fallback-runner: ubuntu-24.04
-          primaries-required: 1
-          fallback-on-error: true
 
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
@@ -87,6 +82,7 @@ jobs:
         id: discover
         env:
           TEST_FILE: ${{ inputs.test_file }}
+          UGNAS_CI_MAX_JOBS: ${{ vars.UGNAS_CI_MAX_JOBS || 3 }}
         run: |
           set -Eeuo pipefail
           mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
@@ -102,10 +98,10 @@ jobs:
     name: ${{ matrix.test.name }}
     needs: discover-tests
     # Opt in trusted branches only; PRs always use GitHub-hosted runners.
-    # UGNAS_CI_TESTS is a JSON array of matrix names, e.g. ["main/test_cli_list"].
-    # The hosted discovery job falls back when workers are offline/busy or the
+    # Ordinary Linux tests are eligible; # ci-runner: hosted declares exceptions.
+    # The hosted discovery job falls back when workers are offline or the
     # availability API fails. UGNAS_CI_ENABLED=false disables the pool entirely.
-    runs-on: ${{ fromJSON(contains(fromJSON(vars.UGNAS_CI_TESTS || '[]'), matrix.test.name) && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
+    runs-on: ${{ fromJSON(matrix.test.ugnas && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
     env:
       CHROME_HEADLESS: "true"
 
@@ -116,8 +112,21 @@ jobs:
 
     steps:
       - name: Set job-local runtime paths
+        env:
+          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
         run: |
+          workers=2
+          cache_suffix=''
+          if [[ "$CI_RUNNER_ENVIRONMENT" == github-hosted ]]; then
+            workers="$(nproc)"
+            cache_suffix="-workers-$workers"
+            # Avoid the hosted image's cold Chromium executable page faults.
+            # abxpkg owns installation and reuse in the existing runtime cache.
+            echo "CHROME_BINPROVIDERS=playwright,puppeteer" >> "$GITHUB_ENV"
+          fi
           {
+            echo "CI_TEST_WORKERS=$workers"
+            echo "ABXPKG_CACHE_WORKERS_SUFFIX=$cache_suffix"
             echo "ABXPKG_LIB_DIR=$RUNNER_TEMP/archivebox-lib"
             echo "DATA_DIR=$RUNNER_TEMP/archivebox-test-data"
             echo "PERSONAS_DIR=$RUNNER_TEMP/abx-personas"
@@ -133,6 +142,7 @@ jobs:
         with:
           python-version: ${{ env.CI_PYTHON_VERSION }}
           architecture: x64
+        id: python
 
       - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
@@ -140,6 +150,41 @@ jobs:
           enable-cache: false
           cache-dependency-glob: uv.lock
 
+      # Self-hosted tool storage survives jobs. Normal abxpkg validation and
+      # install locks allow reuse across files; cold-install tests use tmp_path.
+      - name: Reuse persistent runner tools
+        if: runner.environment == 'self-hosted'
+        env:
+          PERSISTENT_ABXPKG_LIB: ${{ runner.tool_cache }}/abxpkg/${{ github.repository }}/${{ steps.python.outputs.python-version }}
+        run: |
+          mkdir -p "$PERSISTENT_ABXPKG_LIB"
+          echo "ABXPKG_LIB_DIR=$PERSISTENT_ABXPKG_LIB" >> "$GITHUB_ENV"
+
+      # Restored binaries still pass normal abxpkg resolution/version checks.
+      # Installation and isolation tests explicitly choose fresh temporary libs.
+      - name: Identify runtime dependency cache
+        shell: bash
+        run: |
+          uv run --no-project python - <<'PYTHON' >> "$GITHUB_ENV"
+          import hashlib
+          import json
+          import tomllib
+          from pathlib import Path
+
+          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
+          versions = {p["name"]: p["version"] for p in packages if p["name"] in {"abxpkg", "abx-plugins"} and "editable" not in p["source"]}
+          print("ABXPKG_CACHE_DEPS=" + hashlib.sha256(json.dumps(versions, sort_keys=True).encode()).hexdigest())
+          PYTHON
+
+      - name: Reuse validated runtime dependencies
+        if: runner.environment != 'self-hosted'
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ${{ runner.temp }}/archivebox-lib
+          key: abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-${{ env.ABXPKG_CACHE_DEPS }}${{ env.ABXPKG_CACHE_WORKERS_SUFFIX }}
+          restore-keys: |
+            abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-
+
       - name: Resolve optional build dependencies through abxpkg
         if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
         env:
@@ -173,26 +218,38 @@ jobs:
             extra_args=(--extra "$TEST_EXTRA")
           fi
           uv sync --no-cache --locked --dev "${extra_args[@]}"
-      - name: Install Playwright Chromium for browser tests
-        if: contains(matrix.test.paths_arg, 'test_proof_previews') || contains(matrix.test.paths_arg, 'test_snapshot_output_stacks') || contains(matrix.test.paths_arg, 'test_ui_admin_snapshot') || contains(matrix.test.paths_arg, 'test_cli_extract')
-        run: uv run --no-cache --no-sync --no-sources playwright install chromium
-
       - name: Run ${{ matrix.test.name }}
         env:
           TEST_PATHS: ${{ matrix.test.paths_arg }}
+          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
         run: |
           set -Eeuo pipefail
+          echo "Running pytest with $CI_TEST_WORKERS workers on $CI_RUNNER_ENVIRONMENT"
           read -r -a test_paths <<< "$TEST_PATHS"
+          test_args=()
           for test_path in "${test_paths[@]}"; do
             test -f "$test_path"
+            test_args+=("$GITHUB_WORKSPACE/$test_path")
           done
-          for test_path in "${test_paths[@]}"; do
-            test_name="$(basename "$test_path" .py)"
-            test_slug="${test_name//[^A-Za-z0-9_.-]/_}"
-            export DATA_DIR="$RUNNER_TEMP/archivebox-test-data/$test_slug"
-            export PERSONAS_DIR="$RUNNER_TEMP/abx-personas/$test_slug"
-            export CHROME_USER_DATA_DIR="$PERSONAS_DIR/Default/chrome_profile"
-            mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest/$test_slug"
-            uv run --no-cache --no-sync --no-sources pytest -vs "$test_path" \
-              --basetemp="$RUNNER_TEMP/pytest/$test_slug"
-          done
+          # Each test still gets isolated DB/filesystem state from conftest.
+          # Keep one worker pool per batch instead of starting it for every file.
+          mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest"
+          uv run --no-cache --no-sync --no-sources pytest -vs -n "$CI_TEST_WORKERS" "${test_args[@]}" --durations=20 \
+            --basetemp="$RUNNER_TEMP/pytest" \
+            --junitxml="$RUNNER_TEMP/test-results/results.xml"
+      - name: Save test results
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: test-results-${{ strategy.job-index }}
+          path: ${{ runner.temp }}/test-results/*.xml
+          if-no-files-found: error
+          retention-days: 14
+      - name: Save failed runtime diagnostics
+        if: failure()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: runtime-logs-${{ strategy.job-index }}
+          path: ${{ runner.temp }}/pytest/**/*.log
+          if-no-files-found: ignore
+          retention-days: 14

.github/workflows/test.yml

@@ -2,6 +2,12 @@ name: Install and CLI Compatibility
 
 on:
   workflow_call:
+    inputs:
+      platform:
+        description: Platform for install and CLI compatibility
+        required: false
+        default: linux
+        type: string
 
 env:
   DOCKER_IMAGE: archivebox-ci
@@ -19,16 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-24.04
-            os_name: linux
-            python: "3.14"
-          - os: macos-15
-            os_name: macOS
-            python: "3.13"
-          - os: macos-15
-            os_name: macOS
-            python: "3.14"
+        include: ${{ fromJSON(inputs.platform == 'macos' && '[{"os":"macos-15","os_name":"macOS","python":"3.13"},{"os":"macos-15","os_name":"macOS","python":"3.14"}]' || '[{"os":"ubuntu-24.04","os_name":"linux","python":"3.14"}]') }}
 
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

AGENTS.md

@@ -77,11 +77,7 @@ archivebox run
 
 ## Verification
 
-Use targeted tests for focused work:
-
-```bash
-uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
-uv run prek run --all-files
-```
+Run the test suite with `uv run pytest archivebox/tests -x` and repository checks
+with `uv run prek run --all-files`. For focused work, select the relevant tests.
 
 Releases are published only by `.github/workflows/release.yml` after complete CI succeeds for the exact commit. Pushes to `main` prepare stable patch releases automatically; `dev` prepares release candidates. Version selection and dependency repinning are owned by CI and the monorepo coordinator. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.

Dockerfile

@@ -3,14 +3,14 @@
 # Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
 # WHY: abx-dl owns Python, Node, Chromium, and extraction dependencies so the
 # standalone downloader stays independent of ArchiveBox's server features.
-# ArchiveBox-only dependencies (Sonic, OpenCode, supervisor, Django) belong in
+# ArchiveBox-only dependencies (server plugins, supervisor, Django) belong in
 # the layers added by this Dockerfile, never in the shared abx-dl base image.
 # REQUIRED INSTALLATION CONTRACT — do not work around failures:
 # 1. IMAGE CONSTRUCTION: preinstall ALL plugins supported by runtime="archivebox"
 #    and ALL their required_binaries, regardless of default enabled state.
 #    Inherit the complete downloader dependencies from the tested abx-dl image;
 #    install every remaining ArchiveBox-only dependency in this image's layers,
-#    including OpenCode and Sonic. Never put server dependencies in abx-dl.
+#    including optional server plugins. Never put server dependencies in abx-dl.
 #    Use the normal plugin dependency pipeline: install command -> config.json
 #    required_binaries -> abxpkg. Never add manual apt/pip/npm installs to hide
 #    failures in that pipeline. OS/interpreter bootstrap is a separate concern,
@@ -125,6 +125,7 @@ FROM archivebox-runtime-base AS archivebox-builder
 WORKDIR "$CODE_DIR"
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
     --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    --mount=type=bind,source=uv.lock,target=/app/uv.lock \
     <<'EOF'
 echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
 export UV_NO_CACHE=true
@@ -137,48 +138,14 @@ apt-get install -qq -y --no-install-recommends x11-utils
 # Do not rewrite inherited native libraries merely to strip them again.
 /usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print > /tmp/archivebox-inherited-libraries
 
-mkdir -p /tmp/archivebox-uv-project
-/venv/bin/python - <<'PY'
-from pathlib import Path
-import json
-import re
-import urllib.request
-
-source = Path("/app/pyproject.toml")
-target = Path("/tmp/archivebox-uv-project/pyproject.toml")
-text = source.read_text()
-text = text.replace(
-    'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
-    'environments = ["sys_platform == \'linux\'"]',
-)
-
-# Docker builds need the just-published internal abx wheels immediately, but
-# PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a
-# Docker-only dependency view from the version JSON so the published package
-# metadata stays normal while image builds remain resumable after a release.
-for package in ("abxbus", "abxpkg", "abx-plugins", "abx-dl"):
-    match = re.search(
-        rf'"{re.escape(package)}(?P<operator>==|>=)(?P<version>[^"]+)"',
-        text,
-    )
-    if not match:
-        continue
-    version = match.group("version")
-    with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
-        data = json.load(response)
-    wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
-    text = re.sub(
-        rf'"{re.escape(package)}(?:==|>=)[^"]+"',
-        f'"{package} @ {wheel_url}"',
-        text,
-        count=1,
-    )
-
-target.write_text(text)
-PY
-
+# The old Docker-only dependency rewrite predated this checkout's lockfile:
+# it fetched version JSON to avoid stale PyPI Simple listings. That extra API
+# can itself lag or return 503s. The verified lock already records exact public
+# artifact URLs and hashes, including extras and platform markers. Consume it
+# directly, retaining the freshness check instead of resolving a second graph.
 /usr/bin/uv sync \
-    --project /tmp/archivebox-uv-project \
+    --project /app \
+    --locked \
     --no-cache \
     --no-dev \
     --inexact \
@@ -189,6 +156,22 @@ PY
     --no-install-package abx-plugins \
     --no-install-package abx-dl \
     --no-sources
+# Sync may replace inherited dependencies (e.g. pydantic/rich-click), removing
+# their bytecode. Runtime writes are disabled, so missing caches would make
+# every hook compile them again. Compile only missing entries: compileall's
+# timestamp fast path rewrites even valid checked-hash caches, copying the
+# entire inherited venv into this OCI layer for no runtime benefit.
+/usr/bin/uv run --no-project --no-sync python - <<'PY'
+import importlib.util
+from pathlib import Path
+import py_compile
+import sysconfig
+
+for source in Path(sysconfig.get_path("purelib")).rglob("*.py"):
+    cache = Path(importlib.util.cache_from_source(str(source)))
+    if not cache.is_file():
+        py_compile.compile(str(source), doraise=True, invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH)
+PY
 builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
 ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
 /usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
@@ -209,7 +192,7 @@ rm -rf "$builder_abxpkg_lib_dir"
 rm -rf /venv/lib/python3.*/site-packages/pip* \
     /venv/lib/python3.*/site-packages/wheel* \
     /venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel
-rm -rf /var/lib/apt/lists/* /tmp/archivebox-uv-project
+rm -rf /var/lib/apt/lists/*
 EOF
 
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
@@ -283,18 +266,22 @@ RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777
 
 # Install the complete ArchiveBox catalog, including disabled server plugins.
 # Explicit plugin names make installation independent of enabled defaults.
-RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-opencode-$TARGETARCH \
+RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-server-plugins-$TARGETARCH \
     chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" /var/tmp/abxpkg-cache \
     && chmod 1777 /var/tmp/abxpkg-cache \
     && export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
     && env ABXPKG_TMP_CACHE_DIR=/var/tmp/abxpkg-cache XDG_CACHE_HOME=/var/tmp/abxpkg-cache \
         setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install $ABX_DOCKER_PLUGINS \
-    # pnpm includes musl variants that Debian's glibc runtime cannot use.
-    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules" -type l -name 'opencode-linux-*-musl' -delete \
-    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.pnpm" -maxdepth 1 -type d -name 'opencode-linux-*-musl@*' -exec rm -rf {} +
+    && UV_NO_CACHE=true /usr/bin/uv run --no-project python -c 'from abx_plugins.plugins.opencode.image import prune_incompatible_image_files; prune_incompatible_image_files()' \
+    # pnpm's explicit store lives under ABXPKG_LIB_DIR, regardless of XDG_CACHE_HOME.
+    # Retaining its download copies made image size depend on whether the build
+    # preserved hardlinks to installed files. Remove the store in this layer;
+    # installed files retain their contents, and the offline check below proves
+    # the complete catalog still works without the disposable package payloads.
+    && rm -rf "$ABXPKG_LIB_DIR/cache/pnpm"
 RUN --network=none export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups \
-    bash -c '"$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version && abx-dl install $ABX_DOCKER_PLUGINS'
+    bash -c 'uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_installed; verify_installed()" && abx-dl install $ABX_DOCKER_PLUGINS'
 
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
@@ -314,6 +301,9 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \
     && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
 
 RUN for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
+    && test -f "$(/venv/bin/python -c 'import pydantic; print(pydantic.__cached__)')" \
+    && test -f "$(/venv/bin/python -c 'import rich_click; print(rich_click.__cached__)')" \
+    && test -f "$(/venv/bin/python -c 'import platformdirs; print(platformdirs.__cached__)')" \
     && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \

README.md

@@ -105,12 +105,11 @@ curl -fsSL 'https://get.archivebox.io' | bash
 </details>
 <br/>
 
-
 <div align="center" style="text-align: center">
 <br/><br/>
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic"> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/b2765a33-0d1e-4019-a1db-920c7e00e20e" height="75px" alt="logo" align="top"/> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic">
 <br/><br/>
-<small><a href="https://demo.archivebox.io">Demo</a> | <a href="#screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
+<small><a href="https://demo.archivebox.io">Demo</a> | <a href="https://archivebox.io/screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
 <br/>
 <sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
 <br/><br/>
@@ -124,13 +123,13 @@ curl -fsSL 'https://get.archivebox.io' | bash
 ## Key Features
 
 - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE), own your own data & maintain your privacy by self-hosting
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
 - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
-- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage), [Python API](https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
+- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archivedotorg_enabled) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archivedotorg_enabled) for local-only mode)
 - Support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
 - Support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [infiniscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
 
@@ -221,7 +220,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <ol>
 <li>Run the shortcut for the <code>uv</code> install method. It installs <code>uv</code> first when needed.
 <pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | bash</code></pre>
-<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-automatic-setup-script">Install: Bare Metal</a> in the Wiki. ➡️</i>
+<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-uv-install-shortcut">Install: Bare Metal</a> in the Wiki. ➡️</i>
 </li>
 </ol>
 
@@ -381,7 +380,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 > *Warning: These are contributed by external volunteers and may lag behind the official Docker and `uv` channels.*
 
 <ul>
-<li><s>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></s> (<a href="https://truecharts.org/news/scale-deprecation/">TrueCharts is discontinued</a>, wait for <a href="https://forums.truenas.com/t/the-future-of-electric-eel-and-apps/5409/">Electric Eel</a>)</li>
+<li><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a></li>
 <li><a href="https://unraid.net/community/apps?q=archivebox#r">UnRaid</a></li>
 <li><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a></li>
 <li><a href="https://github.com/YunoHost-Apps/archivebox_ynh">Yunohost</a></li>
@@ -432,7 +431,7 @@ For more discussion on third-party hosting options see here: <a href="https://gi
 #### ➡️&nbsp; Next Steps
 
 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
-- (Optional) [Import browser cookies and settings into a persona](docs/Chromium-Install.md#import-an-existing-browser-profile).
+- (Optional) [Import browser cookies and settings into a persona](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile).
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -614,7 +613,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
 <sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
 <br/><br/>
 <a href="https://demo.archivebox.io">DEMO: <code>https://demo.archivebox.io</code></a><br/>
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#Caveats">Caveats</a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#caveats">Caveats</a>
 <br/>
 </div>
 
@@ -642,7 +641,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
   <i>Save directly from the Share menu or Siri/Shortcuts to your server.</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8" height="22px"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files  
-  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file).</i>
+  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#import-a-list-of-urls-from-a-text-file).</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34" height="22px"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)  
   <i>Instructions: <a href="https://support.google.com/chrome/answer/96816?hl=en">Chrome</a>, <a href="https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer">Firefox</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/assets/511499/24ad068e-0fa6-41f4-a7ff-4c26fc91f71a">Safari</a>, <a href="https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows">IE</a>, <a href="https://help.opera.com/en/latest/features/#bookmarks:~:text=Click%20the%20import/-,export%20button,-on%20the%20bottom">Opera</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
@@ -671,7 +670,7 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
 # echo 'https://example.com' | docker compose run --rm -T archivebox add
 ```
 
-See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
+See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage) page for documentation and examples.
 
 It also includes a built-in scheduled import feature with `archivebox schedule`, handled by the same orchestrator that powers `archivebox server`, so you can pull in URLs from RSS feeds and websites regularly without a separate cron container.
 
@@ -812,7 +811,7 @@ Installing directly on <strong>Windows without Docker or WSL/WSL2/Cygwin is not
 
 <h4>Learn More</h4>
 <ul>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Wiki: Install (Dependencies)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">Wiki: Install (Dependencies)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Wiki: Chromium Install</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Wiki: Upgrading or Merging Archives</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Wiki: Troubleshooting (Installing)</a></li>
@@ -866,7 +865,7 @@ Each snapshot subfolder includes static metadata and plain extractor output file
 <h4>Learn More</h4>
 <ul>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Wiki: Setting Up Storage (SMB, NFS, S3, B2, Google Drive, etc.)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
@@ -952,7 +951,7 @@ archivebox manage createsuperuser
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Wiki: Security Overview</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Wiki: Chromium Install (Setting Up a User Profile)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#default_persona">Wiki: Configuration (<code>DEFAULT_PERSONA</code>)</a></li>
 </ul>
 
@@ -1005,7 +1004,7 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active
 
 
 <ul>
-<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
+<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
 <li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
@@ -1066,7 +1065,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <h4>Learn More</h4>
 
 <ul>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user">Wiki: Security Overview (Privilege Dropping)</a></li>
@@ -1191,7 +1190,7 @@ By having users store their content locally or within their organizations, we ca
 
 <h3>Comparison With Other Self-Hosted Archiving Options</h3>
 
-ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
+ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
 
 
 *If you want better fidelity for very complex interactive pages with heavy JS/streams/API requests, check out [ArchiveWeb.page](https://archiveweb.page) and [ReplayWeb.page](https://replayweb.page).*
@@ -1200,7 +1199,7 @@ ArchiveBox differentiates itself from [similar self-hosted projects](https://git
 
 *If you need more advanced recursive spider/crawling ability beyond `--depth=1`, check out [Browsertrix](https://github.com/webrecorder/browsertrix-crawler), [Photon](https://github.com/s0md3v/Photon), or [Scrapy](https://scrapy.org/) and pipe the outputted URLs into ArchiveBox.*
 
-For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects)...
+For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects)...
 
 ArchiveBox is neither the highest fidelity nor the simplest tool available for self-hosted archiving, rather it's a jack-of-all-trades that tries to do most things well by default. We encourage you to try these other tools made by our friends if ArchiveBox isn't suited to your needs.
 
@@ -1255,7 +1254,7 @@ We use the [ArchiveBox GitHub Wiki](https://github.com/ArchiveBox/ArchiveBox/wik
 - [Usage](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage)
 - [Configuration](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)
 - [Supported Sources](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
-- [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox/wiki#can-save-these-things-for-each-site)
+- [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox#output-formats)
 - [Scheduled Archiving](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving)
 
 ## Advanced

archivebox/api/v1_api.py

@@ -2,6 +2,8 @@ __package__ = "archivebox.api"
 
 
 from io import StringIO
+from functools import lru_cache
+from typing import Any
 from traceback import format_exception
 from contextlib import redirect_stdout, redirect_stderr
 
@@ -10,6 +12,7 @@ from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, Permissio
 from django.contrib.auth.models import User
 
 from ninja import NinjaAPI, Swagger
+from ninja.openapi.schema import OpenAPISchema
 
 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
 
@@ -49,6 +52,16 @@ def register_urls(api: NinjaAPI) -> NinjaAPI:
 
 
 class NinjaAPIWithIOCapture(NinjaAPI):
+    def get_openapi_schema(self, *, path_prefix: str | None = None, path_params: dict[str, Any] | None = None) -> OpenAPISchema:
+        prefix = path_prefix if path_prefix is not None else self.get_root_path(path_params or {})
+        return self._schema_for_prefix(prefix)
+
+    @lru_cache(maxsize=8)
+    def _schema_for_prefix(self, path_prefix: str) -> OpenAPISchema:
+        # Routes are registered at startup. Server discovery can reuse their
+        # schema while preserving distinct URL prefixes for mounted instances.
+        return super().get_openapi_schema(path_prefix=path_prefix)
+
     def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
         stdout, stderr = StringIO(), StringIO()
 

archivebox/api/v1_core.py

@@ -211,11 +211,22 @@ class ArchiveResultFilterSchema(FilterSchema):
     created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
 
 
+def _with_archiveresult_processes(queryset):
+    # cmd/pwd/version delegate to Process and Binary. Without this join a full
+    # capture's metadata response performs two extra indexed lookups per hook
+    # (roughly 100 queries), despite the initial Snapshot UUID lookup being fast.
+    # Logs and the hydrated hook environment are not response fields; loading
+    # those potentially large values would replace round trips with excess I/O.
+    return queryset.select_related("process__binary").defer("process__stdout", "process__stderr", "process__env")
+
+
 @router.get("/archiveresults", response=list[ArchiveResultSchema], url_name="get_archiveresult")
 @paginate(CustomPagination)
 def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
     """List all ArchiveResult entries matching these filters."""
-    queryset = filters.filter(ArchiveResult.objects.all())
+    queryset = filters.filter(
+        _with_archiveresult_processes(ArchiveResult.objects.all()).prefetch_related("snapshot__crawl__created_by", "snapshot__tags"),
+    )
     if filters.search or filters.snapshot_tag:
         return queryset.distinct()
     return queryset
@@ -223,26 +234,31 @@ def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterS
 
 def _uuid_ref_query(field_name: str, ref: str) -> Q:
     raw_ref = str(ref or "").strip()
-    query = Q(**{f"{field_name}__startswith": raw_ref})
-    if raw_ref:
-        query |= Q(**{f"{field_name}__icontains": raw_ref})
     try:
         parsed_uuid = UUID(raw_ref)
     except (TypeError, ValueError):
+        query = Q(**{f"{field_name}__startswith": raw_ref})
+        if raw_ref:
+            query |= Q(**{f"{field_name}__icontains": raw_ref})
         normalized_ref = raw_ref.replace("-", "")
         if normalized_ref and normalized_ref != raw_ref:
             query |= Q(**{f"{field_name}__startswith": normalized_ref})
             query |= Q(**{f"{field_name}__icontains": normalized_ref})
-    else:
-        query |= Q(**{field_name: parsed_uuid})
-        query |= Q(**{f"{field_name}__startswith": parsed_uuid.hex})
-    return query
+        return query
+    # An exact UUID cannot need a substring fallback. OR-ing it with LIKE
+    # prevents an indexed lookup and made Cabbage metadata reads take seconds.
+    return Q(**{field_name: parsed_uuid})
 
 
 @router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
 def get_archiveresult(request: HttpRequest, archiveresult_id: str):
     """Get a specific ArchiveResult by id."""
-    return ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
+    return (
+        _with_archiveresult_processes(ArchiveResult.objects.all())
+        .select_related("snapshot__crawl__created_by")
+        .prefetch_related("snapshot__tags")
+        .get(_uuid_ref_query("id", archiveresult_id))
+    )
 
 
 def _normalize_uploaded_archiveresult_plugin(plugin: str) -> str:
@@ -350,6 +366,12 @@ def _summarize_archiveresult_output_files(output_files: dict[str, dict[str, Any]
 
 def _get_snapshot_by_ref(snapshot_id: str):
     queryset = Snapshot.objects.select_related("crawl__created_by")
+    try:
+        full_id = UUID(snapshot_id)
+    except (TypeError, ValueError):
+        pass
+    else:
+        return queryset.get(pk=full_id)
     try:
         return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
     except Snapshot.DoesNotExist:
@@ -392,13 +414,13 @@ def _write_archiveresult_files(
     mime_types = _get_archiveresult_upload_form_values(request, "mime_types", "mime_type")
     chunk_output_path = _get_archiveresult_upload_form_value(request, "chunk_output_path")
 
+    output_files = dict(existing_output_files or {})
+    if not files:
+        return output_files
+
     snapshot_dir = snapshot.output_dir
     plugin_dir = snapshot_dir / plugin_name
     storage = FileSystemStorage(location=str(plugin_dir))
-    output_files = dict(existing_output_files or {})
-
-    if not files:
-        return output_files
 
     if chunk_output_path:
         if len(files) != 1:
@@ -630,6 +652,8 @@ def patch_archiveresult(
 class SnapshotSchema(Schema):
     TYPE: str = "core.models.Snapshot"
     id: UUID
+    crawl_id: UUID
+    persona: str | None
     created_by_id: str
     created_by_username: str
     created_at: datetime
@@ -648,6 +672,17 @@ class SnapshotSchema(Schema):
     num_archiveresults: int
     archiveresults: list[MinimalArchiveResultSchema]
 
+    @staticmethod
+    def resolve_archive_path(obj):
+        return obj.archive_path_from_db
+
+    @staticmethod
+    def resolve_persona(obj):
+        # A deduplicated submission can reuse a capture from a different
+        # persona. Report its actual DB-backed owner, not the new request's choice.
+        persona = obj.crawl.resolve_persona()
+        return persona.name if persona else None
+
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by.pk)
@@ -670,12 +705,14 @@ class SnapshotSchema(Schema):
 
     @staticmethod
     def resolve_num_archiveresults(obj, context):
-        return obj.archiveresult_set.all().distinct().count()
+        # This reverse FK contains each result once. DISTINCT would materialize
+        # wide JSON output columns just to count rows through snapshot_id's index.
+        return obj.archiveresult_set.count()
 
     @staticmethod
     def resolve_archiveresults(obj, context):
         if bool(context["request"].__dict__.get("with_archiveresults", False)):
-            return obj.archiveresult_set.all().distinct()
+            return _with_archiveresult_processes(obj.archiveresult_set.all())
         return ArchiveResult.objects.none()
 
 
@@ -700,6 +737,7 @@ class SnapshotDeleteResponseSchema(Schema):
     snapshot_id: str
     crawl_id: str
     deleted_count: int
+    queued_count: int
 
 
 def normalize_tag_list(tags: list[str] | None = None) -> list[str]:
@@ -938,6 +976,15 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     # lock. The unique insert recovery and CAS update below are the request-side
     # coordination boundary for this idempotent metadata sync.
     snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+    if snapshot is None and bool(crawl.get_current_config().get("ONLY_NEW", True)):
+        # Uploading browser metadata is part of the same submission as /add.
+        # It must honor the runner's deduplication policy, even if it arrives
+        # before the runner has processed that crawl. Report the actual owner
+        # so clients never mistake a reused capture for one they just created.
+        snapshot = Snapshot.objects.filter(url=data.url).exclude(status=Snapshot.DELETING_STATE).order_by("-created_at").first()
+        if snapshot is not None:
+            setattr(request, "with_archiveresults", False)
+            return snapshot
     if snapshot is None:
         try:
             snapshot = Snapshot.objects.create(
@@ -971,11 +1018,6 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
             created_by=request.user if isinstance(request.user, User) else None,
         )
 
-    try:
-        snapshot.ensure_crawl_symlink()
-    except Exception:
-        pass
-
     setattr(request, "with_archiveresults", False)
     return snapshot
 
@@ -1037,22 +1079,23 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
 
 @router.delete("/snapshot/{snapshot_id}", response=SnapshotDeleteResponseSchema, url_name="delete_snapshot")
 def delete_snapshot(request: HttpRequest, snapshot_id: str):
+    """Queue deletion; the normal server runner stops writers and removes files.
+
+    The snapshot remains readable with status=deleting until cleanup succeeds.
+    queued_count reports acceptance; deleted_count is zero in this response.
+    """
     snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
     snapshot_id_str = str(snapshot.id)
-    crawl_id_str = str(snapshot.crawl.pk)
-    snapshot.cancel()
-
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        snapshot = get_snapshot(request, snapshot_id_str, with_archiveresults=False)
-        deleted_count, _ = snapshot.delete()
+    crawl_id_str = str(snapshot.crawl_id)
+    # Use the same durable queue as the admin delete action. Stopping writers
+    # and removing archive files belongs to the runner, never the HTTP request.
+    queued_count = Snapshot.objects.filter(pk=snapshot.pk).request_delete()
     return {
         "success": True,
         "snapshot_id": snapshot_id_str,
         "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
+        "deleted_count": 0,
+        "queued_count": queued_count,
     }
 
 

archivebox/api/v1_crawls.py

@@ -182,6 +182,14 @@ def crawl_file(request: HttpRequest, crawl_id: str, path: str):
     if not file_path.is_file() or crawl_root not in file_path.parents:
         raise HttpError(404, "Crawl file not found")
 
+    if path == "chrome_screencast/latest.jpg" and request.GET.get("after"):
+        try:
+            started_at = float(request.GET["after"])
+        except ValueError:
+            raise HttpError(400, "Invalid screencast start time")
+        if file_path.stat().st_mtime < started_at:
+            raise HttpError(404, "No frame for this capture yet")
+
     response = FileResponse(file_path.open("rb"))
     response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
     response["Pragma"] = "no-cache"

archivebox/api/v1_personas.py

@@ -5,9 +5,11 @@ from datetime import datetime
 from typing import Any, Literal
 from uuid import UUID
 
+from django.core.exceptions import ValidationError
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.errors import HttpError
 from ninja.pagination import paginate
 from pydantic import Field
 
@@ -108,10 +110,9 @@ def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrows
 
 
 def find_persona(extension_persona_id: str, name: str) -> Persona | None:
+    named = Persona.find_named(name)
     return (
-        Persona.objects.filter(
-            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
-        )
+        Persona.objects.filter(Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(pk=named.pk if named else None))
         .order_by("created_at")
         .first()
     )
@@ -138,7 +139,11 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     if not is_valid:
         raise ValueError(error_message)
 
-    persona = find_persona(payload.extension_persona_id, name)
+    try:
+        persona = find_persona(payload.extension_persona_id, name)
+        (persona or Persona(name=name)).validate_name(persona.name if persona else name)
+    except ValidationError as err:
+        raise HttpError(409, "; ".join(err.messages)) from err
     created = persona is None
     if persona is None:
         persona = Persona(name=name)

archivebox/base_models/models.py

@@ -9,6 +9,7 @@ from typing import Any
 from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
 
+from django.core.exceptions import FieldDoesNotExist
 from django.db import models
 from django.db.models import F
 from django.db import transaction
@@ -23,6 +24,110 @@ from django_stubs_ext.db.models import TypedModelMeta
 from archivebox.config import CONSTANTS
 
 
+class PagedQuerySet(models.QuerySet):
+    def paged_iterator(self, chunk_size: int = 500):
+        """
+        Iterate rows using bounded keyset pages instead of one streaming cursor.
+
+        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
+        cursor open until the full queryset is exhausted. That is fine for
+        read-only exports, but update/migration code does filesystem work and
+        writes while iterating; a long-lived read cursor there can stretch lock
+        waits across thousands of rows. This respects the queryset's existing
+        filters, order_by(), select_related(), and prefetch_related() state; if
+        no ordering is defined, it falls back to primary-key order.
+        """
+        pk_field = self.model._meta.pk.name
+        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
+
+        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        ordering = []
+        for term in raw_ordering:
+            descending = term.startswith("-")
+            field_name = term[1:] if descending else term
+            if field_name == "pk":
+                field_name = pk_field
+            ordering.append(f"-{field_name}" if descending else field_name)
+
+        ordered_field_names = [term.removeprefix("-") for term in ordering]
+        try:
+            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
+                offset = 0
+                while True:
+                    batch = list(self[offset : offset + chunk_size])
+                    if not batch:
+                        break
+                    yield from batch
+                    offset += chunk_size
+                return
+        except (AttributeError, FieldDoesNotExist):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
+        if not any(field_name in unique_field_names for field_name in ordered_field_names):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        last_values = None
+        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
+        while True:
+            batch_qs = self.order_by(*ordering)
+            if last_values is not None:
+                page_filter = models.Q()
+                for idx, term in enumerate(ordering):
+                    descending = term.startswith("-")
+                    field_name = term[1:] if descending else term
+                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
+                    comparison = "lt" if descending else "gt"
+                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
+                # Anchor the OR tie-breaks with an explicit leading range so
+                # SQLite/PG can seek the existing order index instead of
+                # rescanning its prefix for every keyset page.
+                first_comparison = "lte" if ordering[0].startswith("-") else "gte"
+                batch_qs = batch_qs.filter(
+                    page_filter,
+                    **{f"{ordered_field_names[0]}__{first_comparison}": last_values[0]},
+                )
+
+            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
+            if not batch_rows:
+                break
+
+            pk_idx = value_field_names.index(pk_field)
+            row_ids = [row[pk_idx] for row in batch_rows]
+            rows_by_id = {obj.pk: obj for obj in self.filter(pk__in=row_ids).order_by()}
+
+            for row in batch_rows:
+                row_id = row[pk_idx]
+                obj = rows_by_id.get(row_id)
+                if obj is not None:
+                    yield obj
+
+            last_values = batch_rows[-1][: len(ordered_field_names)]
+
+
 def normalize_config_json_values(config: Any) -> Any:
     if not isinstance(config, dict):
         return config
@@ -191,9 +296,21 @@ class ModelWithDeleteAfter(models.Model):
         return cls.objects.none()
 
     @classmethod
-    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
+    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True, backfill_cursors: dict | None = None) -> int:
         if backfill_missing:
-            missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
+            # Bound the scanned rows BEFORE evaluating JSON policy/parent
+            # joins. LIMIT on the policy query only bounded its output: finding
+            # no matches could scan the entire history with a SQLite read lock.
+            # The runner keeps a keyset cursor between idle passes, so disabled
+            # policies cannot starve older rows. Restarting safely repeats work.
+            page = cls.objects.order_by("-pk")
+            before = backfill_cursors.get(cls) if backfill_cursors is not None else None
+            if before is not None:
+                page = page.filter(pk__lt=before)
+            page_ids = list(page.values_list("pk", flat=True)[:batch_size])
+            if backfill_cursors is not None:
+                backfill_cursors[cls] = page_ids[-1] if len(page_ids) == batch_size else None
+            missing_delete_at = list(cls.missing_delete_at_candidates().filter(pk__in=page_ids).order_by())
             for obj in missing_delete_at:
                 if obj.set_delete_at_from_config():
                     cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
@@ -231,12 +348,9 @@ class ModelWithOutputDir(ModelWithUUID):
 
     _delete_signal_registered = False
 
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
-        output_dir = Path(self.output_dir)
-        # Avoid holding SQLite write transactions open across slow filesystem work.
-        transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
-        # Note: index.json is deprecated, models should use write_index_jsonl() for full data
+    # Saving metadata must not resolve or create output paths. The runner and
+    # explicit artifact writers create their own directories when writing files;
+    # on_commit would still make HTTP requests wait on remote archive storage.
 
     @property
     def output_dir_parent(self) -> str:
@@ -282,7 +396,14 @@ class ModelWithOutputDir(ModelWithUUID):
             if path.is_symlink() or path.is_file():
                 path.unlink(missing_ok=True)
             elif path.is_dir():
-                shutil.rmtree(path, ignore_errors=True)
+                shutil.rmtree(path)
+            # lstat also catches dangling symlinks and propagates mount and
+            # permission errors rather than mistaking them for absence.
+            try:
+                path.lstat()
+            except FileNotFoundError:
+                continue
+            raise OSError(f"Output path still exists after deletion: {path}")
 
     def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
         """Capture output paths before DB deletion and remove them after commit."""
@@ -294,7 +415,9 @@ class ModelWithOutputDir(ModelWithUUID):
         if cls._delete_signal_registered:
             return
 
-        def schedule_output_dir_cleanup(sender, instance, using, **kwargs):
+        def schedule_output_dir_cleanup(sender, instance, using, origin=None, **kwargs):
+            if getattr(origin, "_output_files_deleted", False):
+                return
             if not isinstance(instance, ModelWithOutputDir):
                 return
             instance.schedule_delete_cleanup(using=using)

archivebox/cli/__init__.py

@@ -2,6 +2,7 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox"
 import os
 import sys
+from contextlib import redirect_stdout
 from importlib import import_module
 
 import rich_click as click
@@ -174,8 +175,12 @@ def cli(ctx, help=False):
                 archivebox_init(quick=True)
                 os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
             check_data_folder()
-            if subcommand != "update":
-                check_migrations(auto_apply=True)
+            if subcommand != "update" and not check_migrations(auto_apply=True):
+                from archivebox.machine.models import Machine
+
+                # Host registration and config reconciliation belong to CLI
+                # startup; get_config() is also used by read-only HTTP views.
+                Machine.current()
         except Exception as e:
             STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
             if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
@@ -215,6 +220,12 @@ def main(args=None, prog_name=None):
         raise SystemExit(130) from None
     finally:
         sys.unraisablehook = previous_unraisablehook
+        supervisor_util = sys.modules.get("archivebox.workers.supervisord_util")
+        if supervisor_util is not None:
+            # Short commands can start a foreground-owned search daemon too.
+            # Finish its cleanup before another CLI can borrow a dying stack.
+            with redirect_stdout(sys.stderr):
+                supervisor_util.stop_own_supervisord_process()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_add.py

@@ -144,7 +144,6 @@ def add(
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
     persona_obj = Persona.get_or_create_named(persona_name)
-    persona_obj.ensure_dirs()
     effective_persona_config = get_config(persona=persona_obj)
 
     crawl_config = {
@@ -217,9 +216,6 @@ def add(
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
-        from archivebox.services.runner import ensure_background_runner
-
-        ensure_background_runner()
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
@@ -427,7 +423,23 @@ def main(**kwargs):
             existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
             kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
 
-        add(urls=urls, **kwargs)
+        from archivebox.machine.models import Process
+
+        # CLI commands own their process record and logs. Queue model saves
+        # must not register a process implicitly: HTTP add() shares that path.
+        process = Process.current()
+        exit_code = 0
+        try:
+            add(urls=urls, **kwargs)
+        except BaseException:
+            exit_code = 1
+            raise
+        finally:
+            # --bg/--index-only return without the foreground runner's cleanup.
+            # Leaving them RUNNING makes the active-process index accumulate
+            # historical CLI commands forever, defeating bounded recovery.
+            if process.status == Process.StatusChoices.RUNNING:
+                process.mark_exited(exit_code=exit_code)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_init.py

@@ -111,6 +111,16 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
         else:
             os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
 
+    from archivebox.machine.models import Machine
+
+    # Registration requires the final schema and reconciles legacy file config.
+    Machine.current()
+
+    from archivebox.personas.models import Persona
+
+    # Initialize here, not when /add/ is rendered: a GET must not create profiles.
+    Persona.get_or_create_default().ensure_dirs()
+
     from archivebox.misc.db import database_display_location, is_postgres
 
     assert database_exists()

archivebox/cli/archivebox_install.py

@@ -58,7 +58,6 @@ def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> Non
     from django.utils import timezone
 
     from archivebox.machine.models import Binary, Machine, _canonical_binary_name
-    from archivebox.services.runner import run_due_binary
 
     machine = Machine.current()
     for requested_name in binary_names:
@@ -80,7 +79,13 @@ def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> Non
             binary.status = Binary.StatusChoices.QUEUED
             binary.retry_at = timezone.now()
             binary.save(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
-        run_due_binary(binary, lock_seconds=60)
+        if binary.status == Binary.StatusChoices.INSTALLED:
+            continue
+        # The synchronous CLI needs the lifecycle's verified installation result,
+        # not the scheduler's "claimed work" result. Reuse its failure/requeue and
+        # health-stat handling so a failed install cannot exit successfully.
+        if not binary.install_claimed(lock_seconds=60):
+            raise click.ClickException(f"Binary {binary_name} is already being installed")
 
 
 def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:

archivebox/cli/archivebox_persona.py

@@ -163,8 +163,9 @@ def create_personas(
             rprint(f'[red]Invalid persona name "{name}": {error_msg}[/red]', file=sys.stderr)
             continue
 
-        persona, created = Persona.objects.get_or_create(
-            name=name,
+        created = Persona.find_named(name) is None
+        persona = Persona.get_or_create_named(
+            name,
             defaults={"config": {"PERMISSIONS": permissions}} if permissions else {},
         )
 
@@ -313,7 +314,9 @@ def update_personas(name: str | None = None) -> int:
             if persona_id:
                 persona = Persona.objects.get(id=persona_id)
             else:
-                persona = Persona.objects.get(name=old_name)
+                persona = Persona.find_named(old_name)
+                if persona is None:
+                    raise Persona.DoesNotExist
 
             # Apply updates from CLI flags
             if name:
@@ -325,7 +328,7 @@ def update_personas(name: str | None = None) -> int:
 
                 # Rename the persona directory too
                 old_path = persona.path
-                persona.name = name
+                persona.name = persona.validate_name(name)
                 new_path = persona.path
 
                 if old_path.exists() and old_path != new_path:

archivebox/cli/archivebox_remove.py

@@ -87,9 +87,8 @@ def remove(
     # Delete one snapshot at a time. Each ``.delete()`` is its own short
     # Django-atomic block, so the writer lock is released between rows and
     # an in-flight daemon transaction can interleave instead of deadlocking.
-    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
-    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
-    # rmtree doesn't hold the lock either.
+    # Snapshot.delete() removes and verifies filesystem outputs first, outside
+    # the short database delete transaction. Failed cleanup retains the row.
     #
     deleted_snapshot_pks = []
     timed_out = False

archivebox/cli/archivebox_server.py

@@ -260,6 +260,12 @@ def server(
     bind_spec = next((arg for arg in runserver_args if arg), "")
     host, port = _parse_and_validate_bind_spec(bind_spec)
 
+    from archivebox.personas.models import Persona
+
+    # Older collections deferred this until /add/. Reconcile before serving any
+    # requests, and report ambiguous profile names before Chrome can use them.
+    Persona.get_or_create_default().ensure_dirs()
+
     if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
         from archivebox.core.routes_util import build_admin_url
 

archivebox/cli/archivebox_snapshot.py

@@ -298,7 +298,7 @@ def list_snapshots(
     Exit codes:
         0: Success (even if no results)
     """
-    from archivebox.misc.jsonl import write_record
+    from archivebox.misc.jsonl import write_records
 
     output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
     if output_formats > 1:
@@ -351,9 +351,7 @@ def list_snapshots(
         return 0
 
     if not is_tty:
-        for snapshot_json in iter_snapshot_json(queryset):
-            write_record(snapshot_json)
-            count += 1
+        count = write_records(iter_snapshot_json(queryset))
         rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
         return 0
 

archivebox/cli/archivebox_version.py

@@ -101,6 +101,7 @@ def _binary_row_dedupe_key(
 def version(
     quiet: bool = False,
     binaries: Iterable[str] = (),
+    no_cache: bool = False,
 ) -> list[str]:
     """Print the ArchiveBox version, debug metadata, and installed dependency versions"""
 
@@ -275,6 +276,7 @@ def version(
             prnt("", f"[yellow]Warning: Could not query collection machine config; resolving through abxpkg: {e}[/yellow]")
 
     declared_binary_specs: dict[str, dict[str, object]] = {}
+    binary_records_by_plugin: dict[str, list[dict[str, object]]] = {}
     for plugin_name, plugin in plugins.items():
         if not plugin_may_have_requested_binary(plugin):
             continue
@@ -286,6 +288,10 @@ def version(
             derived_overrides=derived_config,
             run_output_dir=CONSTANTS.DATA_DIR,
         )
+        # Render the exact requests resolved below. Rehydrating all schemas for
+        # display doubled config work, and could describe different requests if
+        # a config file changed while dependency resolution was running.
+        binary_records_by_plugin[plugin_name] = binary_records
         for binary_record in binary_records:
             actual_name = str(binary_record["name"])
             logical_name = (
@@ -296,9 +302,13 @@ def version(
             display_name = logical_name
             if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
                 continue
-            # This is a diagnostic: recheck the provider instead of trusting an
-            # installation record or a cached version, including disabled plugins.
-            binary_record["no_cache"] = True
+            # Resolve through abxpkg, never the DB's last-known installation row.
+            # Its cache validates current paths, permissions, package fingerprints
+            # and runtime options. Forcing cold resolution here rehashed every
+            # large binary and reran every version subprocess on each diagnostic,
+            # even in an image that had already validated all of them at build.
+            # Keep an explicit full recheck for diagnosing a suspect installation.
+            binary_record["no_cache"] = no_cache
             signature = json.dumps(binary_record, sort_keys=True, default=str)
             declared_binary_specs.setdefault(signature, binary_record)
 
@@ -330,15 +340,9 @@ def version(
             for dependency in plugins.select([plugin_name]).values()
             if not config.get(dependency.enabled_key, True)
         )
-        binary_records = get_required_binary_requests(
-            plugin,
-            plugin.config.required_binaries,
-            overrides=runtime_config,
-            derived_overrides=derived_config,
-            run_output_dir=CONSTANTS.DATA_DIR,
-        )
+        binary_records = binary_records_by_plugin[plugin_name]
         for binary_record in binary_records:
-            binary_record["no_cache"] = True
+            binary_record["no_cache"] = no_cache
             actual_name = str(binary_record["name"])
             logical_name = (
                 Path(actual_name).expanduser().name
@@ -485,6 +489,11 @@ def version(
     is_flag=True,
     help="Only print ArchiveBox version number and nothing else. (equivalent to archivebox --version)",
 )
+@click.option(
+    "--no-cache",
+    is_flag=True,
+    help="Reprobe binary versions and hashes instead of reusing validated provider caches.",
+)
 @click.option(
     "--binaries",
     "-b",

archivebox/cli/rescan.py

@@ -102,9 +102,19 @@ def _reconcile_directory(directory: Path) -> str:
     results = {(item.get("plugin"), item.get("hook_name", "")): item for item in records if item.get("type") == "ArchiveResult"}.values()
     pending = []
     for result in results:
-        result_id = uuid.UUID(result["id"])
         if uuid.UUID(result["snapshot_id"]) != snapshot_id:
             raise ValueError("ArchiveResult belongs to another snapshot")
+        # Legacy JSON conversion produced records without IDs. Recover these
+        # by hook identity while preserving strict collision checks for IDs.
+        if not result.get("id"):
+            if not ArchiveResult.objects.filter(
+                snapshot_id=snapshot_id,
+                plugin=result["plugin"],
+                hook_name=result.get("hook_name", ""),
+            ).exists():
+                pending.append(result)
+            continue
+        result_id = uuid.UUID(result["id"])
         existing = ArchiveResult.objects.filter(pk=result_id).first()
         identity = (snapshot_id, result["plugin"], result.get("hook_name", ""))
         if existing:
@@ -154,7 +164,10 @@ def _reconcile_directory(directory: Path) -> str:
     # Each insert is durable. An interruption after the Snapshot insert is not
     # completion: the next pass still imports missing result IDs and tags.
     for result in pending:
-        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=True)
+        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=bool(result.get("id")))
+    if any(not result.get("id") for result in pending):
+        snapshot.hydrate_archiveresult_output_metadata(snapshot_dir=directory)
+        snapshot.write_index_jsonl(output_dir=directory)
     tags_before = set(snapshot.tags.values_list("name", flat=True))
     snapshot._merge_tags_from_index(record)
     tags_changed = tags_before != set(snapshot.tags.values_list("name", flat=True))
@@ -182,6 +195,10 @@ def reconcile_known_snapshots(snapshots, wait_for_turn=None):
             snapshot.migrate_filesystem_to_current_version()
         directory = Path(snapshot.output_dir)
         if not directory.is_dir():
+            # Canceling queued metadata can seal a row before the runner ever
+            # creates storage. There is nothing on disk to reconcile for it.
+            if snapshot.downloaded_at is None and not snapshot.archiveresult_set.exists():
+                continue
             raise ValueError(f"Missing snapshot directory: {directory}")
         _reconcile_directory(directory)
         for result in snapshot.archiveresult_set.all():

archivebox/config/common.py

@@ -34,6 +34,25 @@ ConfigPayload = dict[str, object]
 PluginSchemaDocuments = dict[str, dict[str, Any]]
 LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 
+
+def _is_cookie_file_config_key(key: str) -> bool:
+    return key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"} or key.endswith("_COOKIES_FILE")
+
+
+def _resolve_cookie_file_paths(config: dict[str, Any]) -> dict[str, Any]:
+    """Keep collection-relative auth paths valid after hooks change cwd."""
+    for key, value in config.items():
+        if _is_cookie_file_config_key(key):
+            if value is None or (isinstance(value, str) and not value.strip()):
+                config[key] = ""
+                continue
+            file_path = Path(value).expanduser()
+            if not file_path.is_absolute():
+                file_path = CONSTANTS.DATA_DIR / file_path
+            config[key] = str(file_path.absolute())
+    return config
+
+
 ###################### Config ##########################
 
 _STDOUT_CONSOLE = Console()
@@ -395,6 +414,36 @@ class ServerConfig(BaseConfigSet):
         )
 
 
+@lru_cache(maxsize=16)
+def _default_sqlite_journal_mode(database_name: str) -> str:
+    if not IN_DOCKER:
+        return "WAL"
+
+    # Native Linux bind mounts share the kernel's locks and mmap; Docker alone
+    # does not make WAL unsafe. DELETE there lets any slow reader block tiny
+    # writes. Desktop VM shares (virtiofs/9p/FUSE), network filesystems and
+    # unknown mounts keep rollback journaling: their host-side readers may not
+    # share SQLite's -shm locking domain. Inspect the DB mount, not archive/.
+    database_path = Path(database_name).expanduser().resolve()
+    try:
+        mounts = Path("/proc/self/mountinfo").read_text().splitlines()
+    except OSError:
+        return "DELETE"
+    filesystem = ""
+    mount_depth = -1
+    for line in mounts:
+        fields, separator, details = line.partition(" - ")
+        if not separator:
+            continue
+        mount_path = fields.split()[4]
+        mount_path = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), mount_path)
+        mount = Path(mount_path)
+        if database_path.is_relative_to(mount) and len(mount.parts) > mount_depth:
+            filesystem = details.split()[0]
+            mount_depth = len(mount.parts)
+    return "WAL" if filesystem in {"ext2", "ext3", "ext4", "xfs", "btrfs", "zfs", "overlay", "tmpfs", "f2fs"} else "DELETE"
+
+
 class DatabaseConfig(BaseConfigSet):
     toml_section_header: str = "DATABASE_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
@@ -410,11 +459,7 @@ class DatabaseConfig(BaseConfigSet):
     DATABASE_USER: str = Field(default="archivebox", alias="ARCHIVEBOX_DATABASE_USER")
     DATABASE_PASSWORD: str = Field(default="", alias="ARCHIVEBOX_DATABASE_PASSWORD")
     SQLITE_JOURNAL_MODE: str = Field(
-        # Docker collections commonly live on a host bind mount. WAL's -shm
-        # locking is only safe when every SQLite process is on the same host;
-        # Docker Desktop/OrbStack place the container and host in different
-        # locking domains and a host-side reader can corrupt the live DB.
-        default="DELETE" if IN_DOCKER else "WAL",
+        default_factory=lambda data: _default_sqlite_journal_mode(str(data["DATABASE_NAME"])),
         alias="ARCHIVEBOX_SQLITE_JOURNAL_MODE",
         pattern=r"(?i)^(DELETE|TRUNCATE|PERSIST|MEMORY|WAL|OFF)$",
     )
@@ -428,11 +473,16 @@ class DatabaseConfig(BaseConfigSet):
     SQLITE_LOCK_RETRY_INTERVAL: float = Field(default=5.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_INTERVAL", gt=0)
 
     @model_validator(mode="after")
-    def reject_docker_sqlite_wal(self):
-        if IN_DOCKER and self.DATABASE_ENGINE.lower() == "sqlite" and self.SQLITE_JOURNAL_MODE.upper() == "WAL":
+    def reject_shared_filesystem_sqlite_wal(self):
+        if (
+            IN_DOCKER
+            and self.DATABASE_ENGINE.lower() == "sqlite"
+            and self.SQLITE_JOURNAL_MODE.upper() == "WAL"
+            and _default_sqlite_journal_mode(self.DATABASE_NAME) != "WAL"
+        ):
             raise ValueError(
-                "SQLITE_JOURNAL_MODE=WAL is unsafe for Docker collections because host bind mounts cross SQLite "
-                "locking domains; use DELETE (the Docker default) or PostgreSQL",
+                "SQLITE_JOURNAL_MODE=WAL requires a local filesystem with shared SQLite locking; "
+                "keep the database on a local Docker volume, use DELETE for host/VM shared mounts, or use PostgreSQL",
             )
         return self
 
@@ -468,6 +518,11 @@ class ArchivingConfig(BaseConfigSet):
     COOKIES_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     AUTH_STORAGE_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
+    @field_validator("COOKIES_FILE", "AUTH_STORAGE_FILE", mode="before")
+    @classmethod
+    def empty_cookie_file_is_unset(cls, value):
+        return None if isinstance(value, str) and not value.strip() else value
+
     URL_DENYLIST: str = Field(
         default=(
             r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$"
@@ -498,7 +553,7 @@ class ArchivingConfig(BaseConfigSet):
             rprint("    (Setting it to somewhere between 30 and 3000 seconds is recommended)", file=sys.stderr)
             rprint(file=sys.stderr)
             rprint("    If you want to make ArchiveBox run faster, disable specific archive methods instead:", file=sys.stderr)
-            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles", file=sys.stderr)
+            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#plugins", file=sys.stderr)
             rprint(file=sys.stderr)
 
     @field_validator("CHECK_SSL_VALIDITY", mode="after")
@@ -825,6 +880,8 @@ class ArchiveBoxBaseConfig(
         if runtime_overrides:
             config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
 
+        _resolve_cookie_file_paths(config)
+
         # Hooks should only see concrete plugin-local flags, never the
         # ArchiveBox selectors used to derive them.
         config.pop("PLUGINS", None)
@@ -849,6 +906,13 @@ class ArchiveBoxBaseConfig(
             lib_dir = CONSTANTS.DATA_DIR / lib_dir
         self.ABXPKG_LIB_DIR = lib_dir.resolve()
 
+        cookie_paths = {key: getattr(self, key) for key in type(self).model_fields if _is_cookie_file_config_key(key)}
+        for key, value in _resolve_cookie_file_paths(cookie_paths).items():
+            if key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"}:
+                setattr(self, key, Path(value) if value else None)
+            else:
+                setattr(self, key, value)
+
         return self
 
     @model_validator(mode="after")
@@ -1113,7 +1177,7 @@ def get_config(
             if apps.ready:
                 from archivebox.machine.models import Machine
 
-                machine = Machine.current()
+                machine = Machine.current_readonly()
         except (DatabaseError, ImportError, RuntimeError, TypeError, ValueError):
             machine = None
 
@@ -1156,11 +1220,15 @@ def get_config(
     scope_overrides: ConfigPayload = {}
 
     if include_machine and machine is not None and machine.config:
-        from archivebox.machine.models import _sanitize_machine_config
+        machine_config = machine.config
+        if resolve_plugins:
+            from archivebox.machine.models import _sanitize_machine_config
+
+            machine_config = _sanitize_machine_config(machine_config, lib_dir=config_data.get("ABXPKG_LIB_DIR"))
 
         scope_overrides.update(
             normalize_runtime_config(
-                _sanitize_machine_config(machine.config, lib_dir=config_data.get("ABXPKG_LIB_DIR")),
+                machine_config,
                 only_crawl_execution=crawl_config_base,
                 exclude_runtime_derived=True,
                 json_safe=False,

archivebox/config/views.py

@@ -150,9 +150,9 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     rows = {
         "Binary Name": [],
-        "Found Version": [],
+        "Recorded Version": [],
         "Provided By": [],
-        "Found Abspath": [],
+        "Recorded Abspath": [],
     }
 
     db_binaries = get_db_binaries_by_name()
@@ -160,18 +160,10 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and binary.is_valid)
-
         rows["Binary Name"].append(ItemLink(name, key=name))
-
-        if binary_is_valid:
-            rows["Found Version"].append(f"✅ {binary.version}" if binary.version else "✅ found")
-            rows["Provided By"].append(binary.binprovider or "-")
-            rows["Found Abspath"].append(binary.abspath or "-")
-        else:
-            rows["Found Version"].append("❌ missing")
-            rows["Provided By"].append("-")
-            rows["Found Abspath"].append("-")
+        rows["Recorded Version"].append(binary.version or "-")
+        rows["Provided By"].append(binary.binprovider or "-")
+        rows["Recorded Abspath"].append(binary.abspath or "-")
 
     return TableContext(
         title="Binaries",

archivebox/core/admin_snapshots.py

@@ -131,11 +131,11 @@ class SnapshotStatusListFilter(admin.SimpleListFilter):
     parameter_name = "snapshot_status"
 
     def lookups(self, request, model_admin):
-        return Snapshot.StatusChoices.choices
+        return Snapshot._meta.get_field("status").choices
 
     def queryset(self, request, queryset):
         value = self.value()
-        if value in Snapshot.StatusChoices.values:
+        if value in dict(Snapshot._meta.get_field("status").choices):
             return queryset.filter(status=value)
         return queryset
 
@@ -992,10 +992,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return format_html(
             '<a class="snapshot-title-detail-hitbox" href="{}" aria-label="Open snapshot details"></a>'
             "{}"
+            "{}"
             '<div class="snapshot-title-url">'
             '<a class="snapshot-original-url" href="{}">{}<code style="user-select: all;">{}</code></a>'
             "</div>",
             detail_url,
+            format_html('<strong class="snapshot-delete-status">{}</strong>', "Deleting — verifying file removal")
+            if obj.status == Snapshot.DELETING_STATE
+            else "",
             title_html,
             url_raw or obj.url,
             favicon_html,
@@ -1535,25 +1539,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.action(
         description="🗑️ Delete",
+        permissions=["delete"],
     )
     def delete_snapshots(self, request, queryset):
-        """Delete snapshots in a single transaction to avoid SQLite concurrency issues."""
-        from django.db import transaction
-
-        total = queryset.count()
-
-        # Get list of IDs to delete first (outside transaction)
-        ids_to_delete = list(queryset.values_list("pk", flat=True))
-
-        # Delete everything in a single atomic transaction
-        with transaction.atomic():
-            deleted_count, _ = Snapshot.objects.filter(pk__in=ids_to_delete).delete()
-
+        """Keep the rows until the runner removes and verifies their files."""
+        total = queryset.request_delete()
         messages.success(
             request,
-            mark_safe(
-                f"Successfully deleted {total} Snapshots ({deleted_count} total objects including related records). Don't forget to scrub URLs from import logs (data/sources) and error logs (data/logs) if needed.",
-            ),
+            f"Queued {total} snapshots for deletion. They remain visible as Deleting until ArchiveBox verifies their files are removed.",
         )
 
     @admin.action(

archivebox/core/asgi.py

@@ -31,7 +31,7 @@ django_application = ArchiveBoxASGIHandler()
 
 async def application(scope, receive, send):
     if scope["type"] == "websocket":
-        from archivebox.opencode.views import websocket_view
+        from abx_plugins.plugins.opencode.archivebox.views import websocket_view
 
         return await websocket_view(scope, receive, send)
     return await django_application(scope, receive, send)

archivebox/core/forms.py

@@ -304,13 +304,13 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             self.can_override_crawl_config = bool(user.is_authenticated and user.is_active and user.is_superuser)
         super().__init__(*args, **kwargs)
 
-        default_persona = Persona.get_or_create_default()
+        # Persona creation belongs to init/server startup, never a GET of /add/.
         persona_queryset = Persona.objects.order_by("name")
         if not self.can_override_crawl_config:
             persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
         self.fields["persona"].queryset = persona_queryset
 
-        selected_persona = persona_queryset.filter(id=default_persona.id).first() or persona_queryset.first()
+        selected_persona = persona_queryset.filter(name__iexact="Default").first() or persona_queryset.first()
         default_config = get_config(persona=selected_persona) if selected_persona else get_config()
         if selected_persona:
             self.fields["persona"].initial = selected_persona.name
@@ -338,6 +338,12 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
                 get_choice_field(self, field_name).choices = [(plugin.name, plugin.name) for plugin in grouped_plugins.get(category, [])]
             self.plugin_groups = []
 
+    def clean_persona(self):
+        persona = self.cleaned_data.get("persona")
+        if persona:
+            persona.validate_name(persona.name)
+        return persona
+
     def clean(self):
         cleaned_data = super().clean() or {}
 

archivebox/core/migrations/0057_repair_migrated_process_links.py

@@ -0,0 +1,25 @@
+"""Repair process links after both core's legacy copy and machine migrations."""
+
+from importlib import import_module
+
+from django.db import migrations
+
+
+repair_process_binary_iface_links = import_module(
+    "archivebox.machine.migrations.0020_repair_process_binary_iface_links",
+).repair_process_binary_iface_links
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("core", "0055_repair_legacy_retry_timestamps"),
+        ("machine", "0022_networkinterface_identity_without_mac"),
+    ]
+
+    operations = [
+        # The original repair can run before core.0027 creates legacy Process
+        # rows. Repair again after the copy, including already upgraded DBs.
+        migrations.RunPython(repair_process_binary_iface_links, migrations.RunPython.noop),
+    ]

archivebox/core/migrations/0058_snapshot_deleting_status.py

@@ -0,0 +1,28 @@
+# Generated by Django 6.1 on 2026-09-30 03:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0057_repair_migrated_process_links"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("queued", "Queued"),
+                    ("started", "Started"),
+                    ("paused", "Paused"),
+                    ("sealed", "Sealed"),
+                    ("deleting", "Deleting"),
+                ],
+                db_index=True,
+                default="queued",
+                max_length=15,
+            ),
+        ),
+    ]

archivebox/core/models.py

@@ -1,6 +1,7 @@
 __package__ = "archivebox.core"
 
 import json
+import logging
 import os
 import uuid
 from collections.abc import Iterable, Mapping, Sequence
@@ -11,7 +12,7 @@ from urllib.parse import urlparse
 
 from django.conf import settings
 from django.contrib import admin
-from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError
+from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import IntegrityError, models, transaction
 from django.db.models import F, Q, QuerySet, Sum, Value
 from django.db.models.fields.json import KT
@@ -23,6 +24,7 @@ from django.utils.safestring import mark_safe
 from django.utils.text import slugify
 
 from archivebox.base_models.models import (
+    PagedQuerySet,
     ModelWithConfig,
     ModelWithDeleteAfter,
     ModelWithHealthStats,
@@ -172,9 +174,28 @@ class SnapshotTag(models.Model):
         unique_together: ClassVar[list[tuple[str, str]]] = [("snapshot", "tag")]
 
 
-class SnapshotQuerySet(models.QuerySet):
+class SnapshotQuerySet(PagedQuerySet):
     """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
 
+    def request_delete(self) -> int:
+        # Admin and API deletion share this durable command. Keep the row until
+        # the runner stops its writers and verifies removal from remote storage;
+        # RETRY_AT_MAX prevents capture scheduling while cleanup is pending.
+        return self.update(status=self.model.DELETING_STATE, retry_at=RETRY_AT_MAX, modified_at=timezone.now())
+
+    def delete(self):
+        # Finish the SELECT before filesystem work, so no read cursor or SQLite
+        # transaction is held while deleting remote archive payloads.
+        snapshots = list(self.select_related("crawl__created_by"))
+        total = 0
+        counts = {}
+        for snapshot in snapshots:
+            deleted, details = snapshot.delete(using=self.db)
+            total += deleted
+            for label, count in details.items():
+                counts[label] = counts.get(label, 0) + count
+        return total, counts
+
     def bulk_create(self, objs, *args, **kwargs):
         objs = list(objs)
         missing_crawl_ids = set()
@@ -206,101 +227,6 @@ class SnapshotQuerySet(models.QuerySet):
                 truncate_overlong_charfields(obj)
         return super().bulk_create(objs, *args, **kwargs)
 
-    def paged_iterator(self, chunk_size: int = 500):
-        """
-        Iterate snapshots using bounded keyset pages instead of one streaming cursor.
-
-        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
-        cursor open until the full queryset is exhausted. That is fine for
-        read-only exports, but update/migration code does filesystem work and
-        writes while iterating; a long-lived read cursor there can stretch lock
-        waits across thousands of rows. This respects the queryset's existing
-        filters, order_by(), select_related(), and prefetch_related() state; if
-        no ordering is defined, it falls back to primary-key order.
-        """
-        pk_field = self.model._meta.pk.name
-        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
-
-        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        ordering = []
-        for term in raw_ordering:
-            descending = term.startswith("-")
-            field_name = term[1:] if descending else term
-            if field_name == "pk":
-                field_name = pk_field
-            ordering.append(f"-{field_name}" if descending else field_name)
-
-        ordered_field_names = [term.removeprefix("-") for term in ordering]
-        try:
-            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
-                offset = 0
-                while True:
-                    batch = list(self[offset : offset + chunk_size])
-                    if not batch:
-                        break
-                    yield from batch
-                    offset += chunk_size
-                return
-        except (AttributeError, FieldDoesNotExist):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
-        if not any(field_name in unique_field_names for field_name in ordered_field_names):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        last_values = None
-        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
-        while True:
-            batch_qs = self.order_by(*ordering)
-            if last_values is not None:
-                page_filter = models.Q()
-                for idx, term in enumerate(ordering):
-                    descending = term.startswith("-")
-                    field_name = term[1:] if descending else term
-                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
-                    comparison = "lt" if descending else "gt"
-                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
-                batch_qs = batch_qs.filter(page_filter)
-
-            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
-            if not batch_rows:
-                break
-
-            pk_idx = value_field_names.index(pk_field)
-            snapshot_ids = [row[pk_idx] for row in batch_rows]
-            snapshots_by_id = {snapshot.pk: snapshot for snapshot in self.filter(pk__in=snapshot_ids).order_by()}
-
-            for row in batch_rows:
-                snapshot_id = row[pk_idx]
-                snapshot = snapshots_by_id.get(snapshot_id)
-                if snapshot is not None:
-                    yield snapshot
-
-            last_values = batch_rows[-1][: len(ordered_field_names)]
-
     # =========================================================================
     # Filtering Methods
     # =========================================================================
@@ -541,6 +467,7 @@ class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ig
 
 class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithQueue):
     BROWSER_EXTENSION_UPLOAD_HOOK_NAME = "on_Snapshot__archivebox_browser_extension_upload"
+    DELETING_STATE = "deleting"
 
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -574,7 +501,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     )
     retry_at = ModelWithQueue.RetryAtField(default=timezone.now)
     status = ModelWithQueue.StatusField(
-        choices=ModelWithQueue.StatusChoices,
+        choices=[*ModelWithQueue.StatusChoices.choices, (DELETING_STATE, "Deleting")],
         default=ModelWithQueue.StatusChoices.QUEUED,
     )
     config = models.JSONField(default=dict, null=False, blank=False, editable=True)
@@ -661,6 +588,49 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def __str__(self):
         return f"[{self.id}] {self.url[:64]}"
 
+    def output_paths_for_delete(self) -> tuple[Path, ...]:
+        current = self.get_storage_path_for_version(self.fs_version)
+        paths = [Path(self.output_dir), current, CONSTANTS.ARCHIVE_DIR / self.timestamp]
+        if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
+            paths.append(current.with_name(str(uuid.UUID(hex=self.id.hex))))
+        if self.crawl_id:
+            domain_dir = self.crawl.output_dir / CONSTANTS.SNAPSHOTS_DIR_NAME / self.extract_domain_from_url(self.url)
+            paths.extend((domain_dir / str(self.id), domain_dir / str(uuid.UUID(hex=self.id.hex))))
+        return tuple(dict.fromkeys(paths))
+
+    def delete(self, using=None, keep_parents=False):
+        from django.db import connections
+
+        using = using or self._state.db or "default"
+        if connections[using].in_atomic_block:
+            raise RuntimeError("Snapshot filesystem deletion must run outside a database transaction")
+        type(self).objects.using(using).filter(pk=self.pk).request_delete()
+        self.status = self.DELETING_STATE
+        from archivebox.machine.models import Process
+
+        process_ids = self.archiveresult_set.filter(process__status=Process.StatusChoices.RUNNING).values_list("process_id", flat=True)
+        for process in list(Process.objects.using(using).filter(pk__in=process_ids)):
+            process.kill_tree()
+            if process.is_running:
+                raise OSError(f"Snapshot output writer is still running: {process.pk}")
+        paths = self.output_paths_for_delete()
+        self.delete_output_paths(paths)
+        # Cascaded ArchiveResults share this directory. Their normal post-commit
+        # cleanup would repeat filesystem work and update an already-deleted row.
+        self._output_files_deleted = True
+        return super().delete(using=using, keep_parents=keep_parents)
+
+    @classmethod
+    def delete_requested(cls, *, batch_size=100):
+        # Finish the bounded read before touching files: even an open SELECT
+        # cursor can hold a SQLite read lock for the duration of slow cleanup.
+        pending = list(cls.objects.filter(status=cls.DELETING_STATE).select_related("crawl__created_by")[:batch_size])
+        for snapshot in pending:
+            try:
+                snapshot.delete()
+            except OSError:
+                logging.getLogger(__name__).exception("Snapshot %s file deletion failed; keeping its database row", snapshot.pk)
+
     @classmethod
     def crawl_count_subquery(cls, *, status: str | None = None, outer_ref: str = "pk") -> QuerySet:
         """Return a scalar subquery counting Snapshots for one outer Crawl."""
@@ -895,7 +865,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             return False
         if self.status == self.StatusChoices.QUEUED:
             return bool(self.url) and self.start_processing()
-        # abx-dl emits SnapshotCompletedEvent after the complete hook sequence;
+        # SnapshotCompletedEvent seals completed or explicitly aborted attempts;
+        # runner takeover preserves the active capture for its replacement.
         # ArchiveResult projection state never drives Snapshot completion.
         return False
 
@@ -910,8 +881,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(config__has_key="DELETE_AFTER") | Q(crawl__config__has_key="DELETE_AFTER"),
+        return (
+            cls.objects.filter(delete_at__isnull=True)
+            .filter(Q(config__has_key="DELETE_AFTER") | Q(crawl__config__has_key="DELETE_AFTER"))
+            .select_related("crawl")
+            .only("id", "created_at", "delete_at", "config", "crawl__config")
         )
 
     @classmethod
@@ -1072,7 +1046,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         from django.db import transaction
 
         def finish_snapshot_save():
-            self.reconcile_filesystem_links()
             crawl = Crawl.objects.filter(pk=self.crawl_id).first()
             if crawl is None:
                 return
@@ -1092,8 +1065,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             # Crawl.urls remains the original submitted source. Snapshot rows
             # are the normalized work queue and discovery projection.
 
-        # get_or_create/update_or_create wrap save() in atomic(); defer filesystem
-        # work and crawl maintenance so SQLite commits before touching the disk.
+        # get_or_create/update_or_create wrap save() in atomic(); attach inherited
+        # tags after that write commits. Filesystem projections belong to the runner.
         transaction.on_commit(finish_snapshot_save)
 
     # =========================================================================
@@ -1197,6 +1170,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         target_dir = self.get_storage_path_for_version(target)
         if target_dir.exists():
+            self.reconcile_with_index(output_dir=target_dir, update_existing_archive_results=False)
             self.hydrate_archiveresult_output_metadata(snapshot_dir=target_dir)
         if cleanup:
             old_dir, new_dir = cleanup
@@ -1229,7 +1203,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def hydrate_archiveresult_output_metadata(self, snapshot_dir: Path | None = None) -> int:
         """Populate missing ArchiveResult file metadata from existing outputs."""
         hydrated = 0
-        for result in self.archiveresult_set.filter(output_files={}).iterator():
+        # Files may live on slow remote storage: finish the bounded read before
+        # inspecting outputs or writing their metadata back to the database.
+        for result in self.archiveresult_set.filter(output_files={}).order_by("pk").paged_iterator():
             hydrated += int(result.update_output_metadata_from_filesystem(snapshot_dir=snapshot_dir))
         return hydrated
 
@@ -3974,6 +3950,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
 
 class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
+    objects = PagedQuerySet.as_manager()
+
     class StatusChoices(models.TextChoices):
         QUEUED = "queued", "Queued"
         STARTED = "started", "Started"
@@ -4195,8 +4173,11 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(snapshot__config__has_key="DELETE_AFTER") | Q(snapshot__crawl__config__has_key="DELETE_AFTER"),
+        return (
+            cls.objects.filter(delete_at__isnull=True)
+            .filter(Q(snapshot__config__has_key="DELETE_AFTER") | Q(snapshot__crawl__config__has_key="DELETE_AFTER"))
+            .select_related("snapshot__crawl")
+            .only("id", "created_at", "delete_at", "snapshot__config", "snapshot__crawl__config")
         )
 
     @property
@@ -4436,6 +4417,10 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
             return 0
 
     def output_file_map(self) -> dict[str, dict[str, Any]]:
+        # List views opt into this cache on their read-only, request-local rows.
+        # Runner-owned results must still reflect metadata changed during a hook.
+        if "_render_output_file_map" in self.__dict__:
+            return self._render_output_file_map
         return self._normalize_output_files(self.output_files)
 
     def output_file_paths(self) -> list[str]:
@@ -4590,11 +4575,20 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
         output_file_map: dict[str, dict[str, Any]] | None = None,
     ) -> str | None:
         ignored = {"stdout.log", "stderr.log", "hook.pid", "listener.pid"}
-        candidates = [
-            path
-            for path in output_file_paths
-            if Path(path).name not in ignored and Path(path).suffix.lower() not in (".pid", ".log", ".sh")
-        ]
+        # Responses/wget manifests can contain thousands of paths. Parse each
+        # once instead of rebuilding Path objects for every preferred filename
+        # and extension group, each time a list row selects a preview.
+        candidates = []
+        names = {}
+        extensions = {}
+        for path in output_file_paths:
+            parsed_path = Path(path)
+            name, extension = parsed_path.name, parsed_path.suffix.lower()
+            if name in ignored or extension in (".pid", ".log", ".sh"):
+                continue
+            candidates.append(path)
+            names.setdefault(name.lower(), path)
+            extensions[path] = extension
         if not candidates:
             return None
 
@@ -4617,9 +4611,8 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
             "article.json",
         ]
         for preferred_name in preferred_names:
-            for candidate in candidates:
-                if Path(candidate).name.lower() == preferred_name:
-                    return candidate
+            if preferred_name in names:
+                return names[preferred_name]
 
         from archivebox.plugins.discovery import get_plugin_output_extension_preference
 
@@ -4630,7 +4623,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
             (".mp4", ".webm", ".mp3", ".opus", ".ogg", ".wav"),
         )
         for ext_group in ext_groups:
-            group_candidates = [candidate for candidate in candidates if Path(candidate).suffix.lower() in ext_group]
+            group_candidates = [candidate for candidate in candidates if extensions[candidate] in ext_group]
             if group_candidates:
                 return max(
                     group_candidates,
@@ -4760,9 +4753,13 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
 
     @property
     def pwd(self) -> str:
-        """Working directory, derived from the snapshot/plugin path if the Process row is gone."""
+        """Working directory metadata without probing archive storage."""
         process = self.process_record
-        return process.pwd if process and process.pwd else str(self.output_dir)
+        return (
+            process.pwd
+            if process and process.pwd
+            else str(self.snapshot.get_storage_path_for_version(self.snapshot.fs_version) / self.plugin)
+        )
 
     @property
     def cmd(self) -> list:

archivebox/core/recovery_util.py

@@ -74,8 +74,9 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
         **snapshot_filter,
     ).update(retry_at=now, modified_at=now)
 
+    # Find rare uploads through hook_name's index before checking their parents;
+    # a correlated upload probe otherwise visits every sealed snapshot.
     extension_upload_results = ArchiveResult.objects.filter(
-        snapshot_id=OuterRef("pk"),
         hook_name=Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME,
     )
     server_results = ArchiveResult.objects.filter(snapshot_id=OuterRef("pk")).exclude(
@@ -83,15 +84,15 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
     )
     extension_only_snapshots = (
         Snapshot.objects.filter(
+            id__in=extension_upload_results.values("snapshot_id"),
             status=Snapshot.StatusChoices.SEALED,
             crawl__status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED],
             **snapshot_filter,
         )
         .annotate(
-            has_extension_upload=Exists(extension_upload_results),
             has_server_result=Exists(server_results),
         )
-        .filter(has_extension_upload=True, has_server_result=False)
+        .filter(has_server_result=False)
     )
     # Older browser-extension uploads could win a race with runner startup:
     # their successful external rows made the fresh Snapshot look finished

archivebox/core/shutdown_util.py

@@ -1,3 +1,28 @@
+"""Shared foreground signal and child-process cleanup contracts.
+
+Shutdown intent and process exit are separate: takeover must preserve the
+replacement owner's work, while a user abort must not later resume the crawl.
+Callers own that policy; this module preserves the received signal and stops
+only the process objects they explicitly supply.
+
+Cooperative boundaries are essential, not just quieter exception handling.
+A real Linux CI regression interrupted Popen.poll() with SIGINT and stranded
+its wait lock. SIGKILL stopped the child (it became a zombie), but Popen.wait()
+still timed out. Raising a different exception or waiting longer cannot repair
+that interrupted critical section. Polling loops must record the signal first
+and honor it between operations; async owners may cancel their own task while
+their loop is running. Once exiting, the last-resort process cleanup must stay
+synchronous: the original loop may be closed, and creating a replacement loop
+cannot recover tasks or resources owned by it.
+
+Keep immediate signal notices and repeated-signal force exit independent of
+the event bus: it may already be draining or closed during shutdown. Resumable
+pause/retry/skip belongs to the existing crawl controller and must not set
+sticky command-shutdown state. See foreground_shutdown_signals below for both
+cooperative and legacy blocking callers; do not change all callers' policy at
+once without supplying their interruption boundaries.
+"""
+
 from __future__ import annotations
 
 import os
@@ -23,7 +48,7 @@ _active_shutdown_state: ShutdownSignalState | None = None
 
 
 def raise_if_shutdown_requested() -> None:
-    """Let long foreground loops honor a signal even if Python ignored it once."""
+    """Honor sticky shutdown intent at a caller-owned interruption boundary."""
 
     if _active_shutdown_state and _active_shutdown_state.signal_name:
         raise KeyboardInterrupt
@@ -48,14 +73,37 @@ def wait_popen_and_kill_children(
     timeout: float,
     kill_timeout: float = 2.0,
 ) -> None:
-    """Wait for a Popen parent and then hard-kill any surviving descendants."""
+    """Reap our own Popen child and stop its explicitly captured descendants.
+
+    A dead child still needs its parent to reap it. Keep wait failures visible;
+    bypassing Popen's lock would hide the interrupted-owner bug described above.
+    Descendants must still be cleaned up when that wait fails.
+    """
 
     try:
-        proc.wait(timeout=timeout)
-    except subprocess.TimeoutExpired:
-        proc.kill()
-        proc.wait(timeout=kill_timeout)
-    kill_remaining_processes(children, timeout=kill_timeout)
+        try:
+            proc.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.wait(timeout=kill_timeout)
+    except subprocess.TimeoutExpired as err:
+        try:
+            state = psutil.Process(proc.pid).as_dict(attrs=["pid", "ppid", "status", "num_threads", "cpu_times"])
+        except psutil.NoSuchProcess:
+            state = {"pid": proc.pid, "status": "exited"}
+        err.add_note(f"Process state after SIGKILL: {state}")
+        # A locked Popen wait can time out even after the OS process exits.
+        # Inspect CPython's lock only for diagnostics; never release it here.
+        waitpid_lock = getattr(proc, "_waitpid_lock", None)
+        err.add_note(
+            f"Popen state: returncode={proc.returncode}, "
+            f"waitpid_lock_locked={waitpid_lock.locked() if waitpid_lock is not None else None}, "
+            f"threads={[thread.name for thread in threading.enumerate()]}",
+        )
+        raise
+    finally:
+        # A parent that cannot be reaped must not leave its descendants running.
+        kill_remaining_processes(children, timeout=kill_timeout)
 
 
 def wait_psutil_and_kill_children(
@@ -104,11 +152,14 @@ def foreground_shutdown_signals(
     interrupt_handlers: dict[signal.Signals, Callable[[], None]] | None = None,
     raise_on_first_signal: bool = True,
 ) -> Iterator[ShutdownSignalState]:
-    """Install foreground signal handlers that print an immediate exit notice.
+    """Keep shutdown intent visible until the owning foreground command unwinds.
 
-    Some log-tail loops intentionally swallow KeyboardInterrupt so that callers
-    can centralize cleanup in finally blocks. The handler writes the signal name
-    immediately, then raises KeyboardInterrupt to break out of the blocking read.
+    Signal state was added because inner loops can swallow KeyboardInterrupt
+    while their caller still needs to stop its children. Cooperative callers use
+    raise_on_first_signal=False and check raise_if_shutdown_requested() between
+    operations, or cancel their owned async task through on_signal. Raising from
+    the handler can otherwise interrupt library locks or partially sent I/O.
+    Blocking callers without a cooperative boundary retain immediate exceptions.
     """
 
     global _active_shutdown_state

archivebox/core/snapshot_status.py

@@ -6,7 +6,7 @@ from django.db.models import QuerySet
 def snapshot_status_values() -> tuple[str, ...]:
     from archivebox.core.models import Snapshot
 
-    return tuple(Snapshot.StatusChoices.values)
+    return tuple(value for value, _label in Snapshot._meta.get_field("status").choices)
 
 
 def normalize_snapshot_status(status: str | None) -> str | None:

archivebox/core/takeover_util.py

@@ -62,7 +62,7 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
         # not keep stealing leadership from the next foreground command.
         base_qs.filter(process_type=Process.TypeChoices.ORCHESTRATOR).exclude(parent__process_type=Process.TypeChoices.SUPERVISORD),
     ):
-        for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+        for proc in qs.order_by("-created_at", "-modified_at").paged_iterator(chunk_size=50):
             if proc.is_running:
                 return proc
             proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
@@ -95,7 +95,7 @@ def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
     )
     if exclude_id is not None:
         qs = qs.exclude(id=exclude_id)
-    for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+    for proc in qs.order_by("-created_at", "-modified_at").paged_iterator(chunk_size=50):
         if proc.is_running:
             return proc
         proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
@@ -178,7 +178,7 @@ def live_runner_processes(*, data_dir: str | Path):
     qs = qs.filter(machine=machine)
     live = []
     foreign_namespace_ids = []
-    for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20):
+    for process in qs.order_by("started_at", "created_at").paged_iterator(chunk_size=20):
         if not process.shares_pid_namespace:
             foreign_namespace_ids.append(process.id)
             continue

archivebox/core/templatetags/core_tags.py

@@ -775,7 +775,9 @@ def snapshot_index_row(context, link) -> str:
     title = unescape(getattr(link, "title", "") or "")
     title_text = title or url
     tags_str = link.tags_str() if callable(getattr(link, "tags_str", None)) else getattr(link, "tags_str", "")
-    tag_html = "".join(f'<span class="snapshot-tag">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag)
+    tag_html = "".join(
+        f'<span class="snapshot-tag" title="{escape(tag)}">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag
+    )
     if tag_html:
         tag_cell = f'<span class="snapshot-tags">{tag_html}</span>'
     else:

archivebox/core/urls.py

@@ -26,15 +26,14 @@ from archivebox.core.views import (
 )
 from archivebox.progressmonitor.views import live_progress_view
 from archivebox.search.views import public_snapshot_search_stream_view
-from archivebox.opencode.views import opencode_proxy_view
+from abx_plugins.plugins.opencode.archivebox.urls import urlpatterns as agent_urlpatterns
 
 urlpatterns = [
     re_path(r"^static/(?P<path>.*)$", serve_static),
     path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}),
     path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}),
     path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"),
-    re_path(r"^admin/agent/?(?=$|opencode)", include("archivebox.opencode.urls")),
-    re_path(r"^(?P<path>assets/.*)$", opencode_proxy_view, name="opencode-assets"),
+    *agent_urlpatterns,
     path("public/search-stream/", public_snapshot_search_stream_view, name="public-search-stream"),
     path("public/", PublicIndexView.as_view(), name="public-index"),
     path("public.html", RedirectView.as_view(url="/public/"), name="public-index-html"),

archivebox/core/views.py

@@ -17,7 +17,6 @@ from django.contrib.auth import HASH_SESSION_KEY, SESSION_KEY, get_user_model
 from django.contrib.auth.mixins import UserPassesTestMixin
 from django.contrib.sessions.models import Session
 from django.core import signing
-from django.core.paginator import InvalidPage
 from django.db.models import Case, IntegerField, Q, Value, When
 from django.http import Http404, HttpRequest, HttpResponse, HttpResponseForbidden, QueryDict
 from django.shortcuts import redirect, render
@@ -48,7 +47,6 @@ from archivebox.config.configset import BaseConfigSet
 from archivebox.core.forms import AddLinkForm
 from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
 from archivebox.core.permissions import (
-    PERMISSIONS_PRIVATE,
     PERMISSIONS_PUBLIC,
     PERMISSIONS_UNLISTED,
     can_view_snapshot,
@@ -92,6 +90,7 @@ from archivebox.plugins.discovery import (
 from archivebox.plugins.forms import get_plugin_config_binary_urls
 from archivebox.plugins.views import get_config_definition_link
 from archivebox.progressmonitor.views import live_progress_view
+from archivebox.progressmonitor.collection import collection_summary
 from archivebox.search.config import (
     get_search_mode,
     get_search_mode_backend,
@@ -123,7 +122,7 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
     if len(lookup) == 12 and "-" not in lookup:
         # Resolve suffixes using the covering primary-key index before fetching
         # wide snapshot rows and joining their crawl/user data.
-        matches = list(Snapshot.objects.filter(id__endswith=lookup).order_by().values_list("id", flat=True))
+        matches = Snapshot.objects.filter(id__endswith=lookup).order_by().values_list("id", flat=True)
         return snapshots.filter(pk__in=matches).order_by("-created_at", "-downloaded_at").first()
 
     try:
@@ -507,7 +506,7 @@ class SnapshotView(View):
         def _resolve_snapshots_for_slug(slug: str):
             # full URLs go straight to the url-only path (fast, indexed)
             if "://" in slug:
-                return SnapshotView.find_snapshots_for_url(slug)
+                return SnapshotView.find_snapshots_for_url(slug, allow_fallback=False)
             # short uuid-shaped slugs (>=8 hex chars after stripping non-hex) try id matching first
             id_qs = SnapshotView.find_snapshots_for_id(slug)
             if id_qs.exists():
@@ -518,6 +517,12 @@ class SnapshotView(View):
         try:
             if "://" in path:
                 snapshot = snapshots.order_by("-bookmarked_at").first()
+                if snapshot is None:
+                    # Fetch the exact match once before trying legacy URL
+                    # fallbacks; an exists() probe followed by the same lookup
+                    # needlessly doubled reads on every normal redirect.
+                    snapshots = direct_snapshots_queryset(request, self.find_snapshots_for_url(path)).select_related("crawl__created_by")
+                    snapshot = snapshots.order_by("-bookmarked_at").first()
                 if snapshot is None:
                     raise Snapshot.DoesNotExist
             else:
@@ -1213,11 +1218,22 @@ class OriginalDomainReplayView(View):
 
 
 class PublicIndexView(ListView):
+    """Keep browsing responsive while the same small server is archiving.
+
+    Performance takes priority over perfect initial totals: on a 1 vCPU / 1 GB
+    host with hundreds of thousands of snapshots, even an indexed count can
+    consume the entire roughly 0.5-0.7 second page-load budget. Benchmark with
+    the runner active, since it shares the CPU, memory, and SQLite database.
+
+    The unfiltered list therefore renders rows first and accepts missing/stale
+    display totals. A later ordinary reload can show the background-cached
+    count; do not restore a synchronous COUNT or add polling just to fill it in.
+    """
+
     template_name = "public_index.html"
     model = Snapshot
     ordering: ClassVar[list[str]] = ["-bookmarked_at", "-created_at"]
     paginator_class = AcceleratedPaginator
-    public_page_scan_chunk_size = 50
 
     def get_paginate_by(self, queryset):
         runtime_config = self.__dict__.get("runtime_config")
@@ -1238,43 +1254,76 @@ class PublicIndexView(ListView):
             "status",
             "output_size",
             "permissions",
+            # Modern replay URLs need the filesystem version and owner's name.
+            # Deferring these caused four extra SELECTs per displayed snapshot.
+            "fs_version",
+            "crawl__created_by__username",
         )
 
-    def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot] | None:
-        target_count = page_number * page_size
-        public_snapshots: list[Snapshot] = []
-        scanned = 0
-        chunk_size = max(self.public_page_scan_chunk_size, page_size)
-        ordered_snapshots = Snapshot.objects.order_by(*self.ordering).only(*self._base_public_snapshot_fields())
-
-        while len(public_snapshots) < target_count:
-            chunk = list(ordered_snapshots[scanned : scanned + chunk_size])
-            if not chunk:
-                break
-            scanned += len(chunk)
-            public_snapshots.extend(snapshot for snapshot in chunk if snapshot.permissions == PERMISSIONS_PUBLIC)
-
+    def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot]:
+        # Walk the existing chronological index in bounded keyset pages,
+        # reading only identity/visibility until reaching the requested page.
+        # Repeated OFFSET batches used to revisit every earlier row AND join
+        # owners each time, making deep pages quadratic. Filtering visibility
+        # in SQL instead made SQLite sort the entire permissions index first.
+        # Each helper page closes its cursor before yielding; only the final
+        # displayed IDs load wide rows/owners. No new index or SQLite hint.
         start = (page_number - 1) * page_size
-        return public_snapshots[start:target_count]
+        visible_count = 0
+        page_ids = []
+        candidates = Snapshot.objects.only("id", "permissions").order_by(*self.ordering, "id")
+        for snapshot in candidates.paged_iterator(chunk_size=max(100, page_size + 1)):
+            if snapshot.permissions != PERMISSIONS_PUBLIC:
+                continue
+            if visible_count >= start:
+                page_ids.append(snapshot.pk)
+                if len(page_ids) > page_size:
+                    break
+            visible_count += 1
+        return list(
+            Snapshot.objects.filter(pk__in=page_ids, permissions=PERMISSIONS_PUBLIC)
+            .select_related("crawl__created_by")
+            .order_by(*self.ordering, "id")
+            .only(*self._base_public_snapshot_fields()),
+        )
 
     def paginate_queryset(self, queryset, page_size):
         if self.request.GET.get("q", default="").strip():
             return super().paginate_queryset(queryset, page_size)
 
-        public_count = self.get_exact_public_snapshot_count()
-        paginator = self.get_paginator(range(public_count), page_size)
         page_kwarg = self.kwargs.get(self.page_kwarg)
         page_query = self.request.GET.get(self.page_kwarg)
         page_number = page_kwarg or page_query or 1
 
         try:
-            page = paginator.page(page_number)
-        except InvalidPage as err:
+            page_number = int(page_number)
+            if page_number < 1:
+                raise ValueError("Page must be positive")
+        except (ValueError, TypeError) as err:
             raise Http404(f"Invalid page ({page_number}): {err}") from err
 
-        object_list = self._ordered_public_page_from_order_index(page_number=page.number, page_size=page_size)
-        page.object_list = object_list
-        return paginator, page, object_list, page.has_other_pages()
+        snapshots = self._ordered_public_page_from_order_index(page_number=page_number, page_size=page_size)
+        if not snapshots and page_number > 1:
+            raise Http404(f"Invalid page ({page_number}): No snapshots")
+        # Exhausting the rows gives an exact total for this read at no extra
+        # cost. Otherwise the lookahead is only a lower bound: an absent count
+        # is preferable to an on-request scan or a cached total smaller than
+        # the rows we just observed. Cached overestimates are acceptable here.
+        observed_count = (page_number - 1) * page_size + len(snapshots)
+        self.public_snapshot_count = observed_count
+        if len(snapshots) > page_size:
+            summary = collection_summary(self.request.user, public=True)
+            cached_count = summary["snapshots"] if summary is not None else None
+            self.public_snapshot_count = cached_count if cached_count is not None and cached_count >= observed_count else None
+
+        # Give Django an in-memory lower bound, never the collection QuerySet
+        # whose Paginator.count would issue COUNT(*). Navigation uses this read's
+        # lookahead even when a stale total would hide a newly added page.
+        # The template must use the separate display totals, not num_pages here.
+        paginator = self.get_paginator(range(observed_count), page_size)
+        page = paginator.page(page_number)
+        page.object_list = snapshots[:page_size]
+        return paginator, page, page.object_list, page.has_other_pages()
 
     def get_context_data(self, **kwargs):
         runtime_config = self.__dict__.get("runtime_config")
@@ -1303,10 +1352,16 @@ class PublicIndexView(ListView):
             and search_mode_backend
             and context["paginator"].count == 0,
         )
+        context["public_snapshot_count"] = context["paginator"].count if query else self.public_snapshot_count
+        count = context["public_snapshot_count"]
+        context["public_page_count"] = (
+            max(1, (count + context["paginator"].per_page - 1) // context["paginator"].per_page) if count is not None else None
+        )
         snapshots = list(context.get("object_list") or ())
         from archivebox.plugins.output_groups import plugin_output_sizes
 
         all_results_by_snapshot = {str(snapshot.id): [] for snapshot in snapshots}
+        snapshots_by_id = {str(snapshot.id): snapshot for snapshot in snapshots}
         icons_by_snapshot: dict[str, dict[str, ArchiveResult]] = {str(snapshot.id): {} for snapshot in snapshots}
         tag_names_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
         list_icon_paths_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
@@ -1343,8 +1398,17 @@ class PublicIndexView(ListView):
                     "output_size",
                 )
             )
-            for result in results.iterator(chunk_size=1000):
+            # This is already bounded by the displayed snapshot page. Finish
+            # the SELECT before parsing output manifests: a streaming cursor
+            # otherwise keeps SQLite's read lock throughout rendering.
+            for result in list(results):
                 snapshot_key = str(result.snapshot_id)
+                # Thumbnail URLs traverse result.snapshot too; share the page's
+                # already-loaded parent rather than fetching it once per result.
+                result.snapshot = snapshots_by_id[snapshot_key]
+                # These read-only page objects are shared by icons, sizes, and
+                # thumbnails. Normalize each manifest once for this render.
+                result._render_output_file_map = result.output_file_map()
                 all_results_by_snapshot[snapshot_key].append(result)
                 progress = progress_by_snapshot[snapshot_key]
                 progress["total"] += 1
@@ -1376,13 +1440,12 @@ class PublicIndexView(ListView):
         context["object_list"] = snapshots
         return context
 
-    def get_exact_public_snapshot_count(self) -> int:
-        hidden_count = Snapshot.objects.filter(permissions=PERMISSIONS_PRIVATE).count()
-        hidden_count += Snapshot.objects.filter(permissions=PERMISSIONS_UNLISTED).count()
-        return Snapshot.objects.count() - hidden_count
-
     def get_queryset(self, **kwargs):
-        qs = public_snapshots_queryset(super().get_queryset(**kwargs)).only(*self._base_public_snapshot_fields())
+        qs = (
+            public_snapshots_queryset(super().get_queryset(**kwargs))
+            .select_related("crawl__created_by")
+            .only(*self._base_public_snapshot_fields())
+        )
         query = self.request.GET.get("q", default="").strip()
 
         if not query:

archivebox/core/widgets.py

@@ -717,7 +717,7 @@ class InlineTagEditorWidget(TagEditorWidget):
                 )
             pills_html += f'''
                 <span class="tag-pill" data-tag="{self._escape(td["name"])}" data-tag-id="{td["id"]}" style="{self._tag_style(td["name"])}">
-                    <a href="/admin/core/snapshot/?tags__id__exact={td["id"]}" class="tag-link">{self._escape(td["name"])}</a>
+                    <a href="/admin/core/snapshot/?tags__id__exact={td["id"]}" class="tag-link" title="{self._escape(td["name"])}">{self._escape(td["name"])}</a>
                     {remove_button}
                 </span>
             '''

archivebox/crawls/admin.py

@@ -947,7 +947,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         updated = 0
         batch = []
         crawls_to_update = []
-        for crawl in queryset.only("id", "config", "permissions").iterator(chunk_size=500):
+        for crawl in queryset.only("id", "config", "permissions").order_by("pk").paged_iterator(chunk_size=500):
             old_permissions = crawl.permissions
             config = dict(crawl.config or {})
             config["PERMISSIONS"] = permissions

archivebox/crawls/models.py

@@ -23,6 +23,7 @@ from archivebox.config.common import rprint as print
 from archivebox.core.permissions import PERMISSIONS_VALUES, normalize_permissions
 
 from archivebox.base_models.models import (
+    PagedQuerySet,
     ModelWithUUID,
     ModelWithDeleteAfter,
     ModelWithOutputDir,
@@ -148,6 +149,8 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
 
 
 class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithQueue):
+    objects = PagedQuerySet.as_manager()
+
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
@@ -229,7 +232,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         if paused and save and self.pk:
             from archivebox.core.models import Snapshot
 
-            for snapshot in self.snapshot_set.exclude(status__in=Snapshot.FINAL_STATES).iterator():
+            for snapshot in self.snapshot_set.exclude(status__in=Snapshot.FINAL_STATES).order_by("pk").paged_iterator():
                 snapshot.pause()
         return paused
 
@@ -301,7 +304,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True, config__has_key="DELETE_AFTER")
+        return cls.objects.filter(delete_at__isnull=True, config__has_key="DELETE_AFTER").only("id", "created_at", "delete_at", "config")
 
     def save(self, *args, **kwargs):
         update_fields = kwargs.get("update_fields")
@@ -368,7 +371,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         batch = []
         updated = 0
         queryset = self.snapshot_set.filter(Q(permissions=old_permissions) | Q(permissions__isnull=True)).only("id", "config")
-        for snapshot in queryset.iterator(chunk_size=500):
+        for snapshot in queryset.order_by("pk").paged_iterator(chunk_size=500):
             config = dict(snapshot.config or {})
             config["PERMISSIONS"] = normalized_new_permissions
             snapshot.config = config
@@ -427,7 +430,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=added_names)}
 
             tag_ids = [tag.pk for tag_name in added_names if (tag := tags_by_name.get(tag_name))]
-            snapshot_ids = Snapshot.objects.filter(crawl=self).values_list("id", flat=True).iterator(chunk_size=5000)
+            # Fetch each page before writing M2M rows. iterator(chunk_size=...)
+            # would keep one read cursor open across every bulk write.
+            snapshot_ids = (
+                snapshot.pk for snapshot in Snapshot.objects.filter(crawl=self).only("id").order_by("pk").paged_iterator(chunk_size=500)
+            )
             while True:
                 batch_snapshot_ids = list(islice(snapshot_ids, 5000))
                 if not batch_snapshot_ids:

archivebox/machine/migrations/0023_repair_binary_field_indexes.py

@@ -0,0 +1,27 @@
+from django.db import migrations, models
+
+
+def repair_binary_field_indexes(apps, schema_editor):
+    Binary = apps.get_model("machine", "Binary")
+    connection = schema_editor.connection
+    with connection.cursor() as cursor:
+        constraints = connection.introspection.get_constraints(cursor, Binary._meta.db_table)
+    indexed_columns = {
+        tuple(constraint["columns"])
+        for constraint in constraints.values()
+        if constraint.get("index") or constraint.get("unique") or constraint.get("primary_key")
+    }
+    # 0005's legacy raw-SQL table creation omitted field indexes even though
+    # migration state declares them. Editing 0005 alone cannot repair already
+    # upgraded collections. Restore only missing single-field indexes, checking
+    # columns rather than generated names so fresh SQLite/PostgreSQL installs
+    # retain their existing indexes without duplicates or a schema-state change.
+    for field in Binary._meta.local_fields:
+        if field.db_index and not field.unique and (field.column,) not in indexed_columns:
+            schema_editor.add_index(Binary, models.Index(fields=[field.name], name=f"mach_binary_{field.column}_idx"))
+
+
+class Migration(migrations.Migration):
+    dependencies = [("machine", "0022_networkinterface_identity_without_mac")]
+
+    operations = [migrations.RunPython(repair_binary_field_indexes, migrations.RunPython.noop)]

archivebox/machine/models.py

@@ -20,7 +20,7 @@ from django.utils.functional import cached_property
 
 from archivebox.config import CONSTANTS
 from archivebox.config.common import rprint
-from archivebox.base_models.models import ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
+from archivebox.base_models.models import PagedQuerySet, ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
 from archivebox.workers.models import ModelWithQueue
 from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats
 
@@ -48,7 +48,6 @@ _CURRENT_PROCESS: Process | None = None
 MACHINE_RECHECK_INTERVAL = 7 * 24 * 60 * 60
 NETWORK_INTERFACE_RECHECK_INTERVAL = 1 * 60 * 60
 BINARY_RECHECK_INTERVAL = 1 * 30 * 60
-PROCESS_RECHECK_INTERVAL = 60  # Re-validate every 60 seconds
 PID_REUSE_WINDOW = timedelta(hours=24)  # Max age for considering a PID match valid
 PROCESS_TIMEOUT_GRACE = timedelta(seconds=30)  # Extra margin before force-cleaning timed-out RUNNING rows
 START_TIME_TOLERANCE = 5.0  # Seconds tolerance for start time matching
@@ -221,35 +220,21 @@ class Machine(ModelWithHealthStats):
         app_label = "machine"
 
     @classmethod
-    def current(cls, refresh: bool = False) -> Machine:
-        global _CURRENT_MACHINE
-        if refresh:
-            _CURRENT_MACHINE = None
-        if _CURRENT_MACHINE:
-            if timezone.now() < _CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
-                # Reconciliation can save() and clear the shared cache; keep
-                # this call's instance so we never return None after that save.
-                machine = _CURRENT_MACHINE
-                # One-time-per-process reconciliation between ArchiveBox.conf
-                # and Machine.config. Fast-path: bool check + early-return when
-                # the sync has already run, so the cached-machine return path
-                # stays sub-microsecond.
-                try:
-                    from archivebox.config.collection import sync_machine_and_file
+    def current_readonly(cls) -> Machine | None:
+        """Read the current machine without registration, refresh, or config writes."""
+        if _CURRENT_MACHINE is not None:
+            return _CURRENT_MACHINE
+        return cls.objects.filter(guid=get_host_guid()).first()
 
-                    sync_machine_and_file(machine)
-                except Exception:
-                    pass
-                return machine
-            else:
-                _CURRENT_MACHINE = None
+    @classmethod
+    def current(cls) -> Machine:
+        global _CURRENT_MACHINE
+        if _CURRENT_MACHINE is not None:
+            return _CURRENT_MACHINE
 
         host_guid = get_host_guid()
-        # save() intentionally invalidates _CURRENT_MACHINE so config edits
-        # made through other instances become visible in this process. Host
-        # refresh (after seven days), sanitation, and reconciliation can all
-        # save here, so the working instance must not live only in that cache.
-        # Otherwise a routine refresh leaves startup dereferencing None.
+        # Host detection, sanitation, and config reconciliation run only on
+        # initial registration, never on cached reads.
         try:
             machine = cls.objects.get(guid=host_guid)
         except cls.DoesNotExist:
@@ -269,44 +254,41 @@ class Machine(ModelWithHealthStats):
                 **get_vm_info(),
                 stats=get_host_stats(),
             )
-        else:
-            if timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
-                for key, value in {
-                    "hostname": socket.gethostname(),
-                    **get_os_info(),
-                    **get_vm_info(),
-                    "stats": get_host_stats(),
-                }.items():
-                    setattr(machine, key, value)
-                machine.save(
-                    update_fields=[
-                        "hostname",
-                        "hw_in_docker",
-                        "hw_in_vm",
-                        "hw_manufacturer",
-                        "hw_product",
-                        "hw_uuid",
-                        "os_arch",
-                        "os_family",
-                        "os_platform",
-                        "os_release",
-                        "os_kernel",
-                        "stats",
-                        "modified_at",
-                    ],
-                )
-        machine = cls._sanitize_config(machine)
-        # Same one-time sync as the cached-return path. Triggers here on the
-        # very first ``Machine.current()`` call in a process before any
-        # cached return can occur.
+        needs_recheck = timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL)
+        # Reconcile before any save can mirror stale DB config over file edits.
         try:
             from archivebox.config.collection import sync_machine_and_file
 
             sync_machine_and_file(machine)
         except Exception:
             pass
-        # Publish only after all save-capable steps; publishing earlier would
-        # let those steps invalidate the cache we are in the middle of filling.
+        if needs_recheck:
+            for key, value in {
+                "hostname": socket.gethostname(),
+                **get_os_info(),
+                **get_vm_info(),
+                "stats": get_host_stats(),
+            }.items():
+                setattr(machine, key, value)
+            machine.save(
+                update_fields=[
+                    "hostname",
+                    "hw_in_docker",
+                    "hw_in_vm",
+                    "hw_manufacturer",
+                    "hw_product",
+                    "hw_uuid",
+                    "os_arch",
+                    "os_family",
+                    "os_platform",
+                    "os_release",
+                    "os_kernel",
+                    "stats",
+                    "modified_at",
+                ],
+            )
+        machine = cls._sanitize_config(machine)
+        # Publish only after initialization is complete.
         _CURRENT_MACHINE = machine
         return machine
 
@@ -377,18 +359,15 @@ class Machine(ModelWithHealthStats):
             if update_fields is not None:
                 kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
 
-        # Drop the ``Machine.current()`` module-level cache on every save so
-        # config edits (admin form, from_json, etc.) become live in the same
-        # process without waiting out the 7-day ``MACHINE_RECHECK_INTERVAL``.
-        # The cache reads ``_CURRENT_MACHINE.modified_at`` and that value
-        # never moves forward on the cached object even when the row is
-        # updated in the DB, so without this we'd keep serving stale
-        # ``machine.config`` (incl. ``BASE_URL``) until the worker restarts.
+        # Apply explicit edits to the cached instance without making the next
+        # read repeat host detection, sanitation, or config reconciliation.
         update_fields = kwargs.get("update_fields")
         super().save(*args, **kwargs)
         global _CURRENT_MACHINE
         if _CURRENT_MACHINE is not None and _CURRENT_MACHINE.pk == self.pk:
-            _CURRENT_MACHINE = None
+            for field in self._meta.concrete_fields:
+                if field.attname in self.__dict__ and (update_fields is None or field.name in update_fields):
+                    setattr(_CURRENT_MACHINE, field.attname, self.__dict__[field.attname])
 
         # Mirror Machine.config into ArchiveBox.conf so the two stores stay
         # 1:1. Skipped when ``update_fields`` is set and doesn't touch
@@ -443,7 +422,7 @@ class NetworkInterface(ModelWithHealthStats):
     @classmethod
     def current(cls, refresh: bool = False) -> NetworkInterface:
         global _CURRENT_INTERFACE
-        machine = Machine.current(refresh=refresh)
+        machine = Machine.current()
         if _CURRENT_INTERFACE and _CURRENT_INTERFACE.machine_id == machine.id:
             if not refresh:
                 # Callers that pass refresh=False are asking for attribution to
@@ -493,7 +472,7 @@ class NetworkInterface(ModelWithHealthStats):
         return _CURRENT_INTERFACE
 
 
-class BinaryManager(models.Manager):
+class BinaryManager(models.Manager.from_queryset(PagedQuerySet)):
     def get_from_db_or_cache(self, name: str, abspath: str = "", version: str = "", sha256: str = "", binprovider: str = "env") -> Binary:
         """Get or create an Binary record from the database or cache."""
         cached = _CURRENT_BINARIES.get(name)
@@ -910,7 +889,7 @@ class Binary(ModelWithHealthStats, ModelWithQueue):
 # =============================================================================
 
 
-class ProcessManager(models.Manager):
+class ProcessManager(models.Manager.from_queryset(PagedQuerySet)):
     """Manager for Process model."""
 
     def current(self) -> Process:
@@ -1229,8 +1208,11 @@ class Process(ModelWithDeleteAfter, models.Model):
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(env__has_key="DELETE_AFTER") | Q(machine__config__has_key="DELETE_AFTER"),
+        return (
+            cls.objects.filter(delete_at__isnull=True)
+            .filter(Q(env__has_key="DELETE_AFTER") | Q(machine__config__has_key="DELETE_AFTER"))
+            .select_related("machine")
+            .only("id", "created_at", "delete_at", "env", "machine__config")
         )
 
     # Properties that delegate to related objects
@@ -1430,6 +1412,13 @@ class Process(ModelWithDeleteAfter, models.Model):
     # Process.current() and hierarchy methods
     # =========================================================================
 
+    @classmethod
+    def current_readonly(cls) -> Process | None:
+        """Return this process's registered identity without registration or probes."""
+        if _CURRENT_PROCESS is not None and _CURRENT_PROCESS.pid == os.getpid():
+            return _CURRENT_PROCESS
+        return None
+
     @classmethod
     def current(cls) -> Process:
         """
@@ -1447,40 +1436,19 @@ class Process(ModelWithDeleteAfter, models.Model):
 
         current_pid = os.getpid()
 
-        # Fast path used by model save diagnostics and hot runner loops. A
-        # cached Process object is valid when the immutable identity we wrote
-        # at creation time still describes this Python process. PID reuse cannot
-        # happen while this process is alive, so pid + present started_at/cmd is
-        # enough here; the slower psutil validation below remains the fallback
-        # for missing/stale cache.
-        if (
-            _CURRENT_PROCESS
-            and _CURRENT_PROCESS.pid == current_pid
-            and _CURRENT_PROCESS.status == cls.StatusChoices.RUNNING
-            and timezone.now() < _CURRENT_PROCESS.modified_at + timedelta(seconds=PROCESS_RECHECK_INTERVAL)
-            and _CURRENT_PROCESS.started_at is not None
-            and bool(_CURRENT_PROCESS.cmd)
-        ):
-            return _CURRENT_PROCESS
+        # This is process identity, not a health check. A live Python process
+        # cannot reuse its own PID, and a fork fails current_readonly's PID
+        # check. Expiring this cache by modified_at made every call after 60s
+        # query the database and touch logs until something else saved the row.
+        # Explicit lifecycle/recovery code owns liveness and interface refresh.
+        cached = cls.current_readonly()
+        if cached is not None:
+            return cached
+        _CURRENT_PROCESS = None
 
         machine = Machine.current()
         iface = NetworkInterface.current()
 
-        # Check cache validity
-        if _CURRENT_PROCESS:
-            # Verify: same PID, same machine, cache not expired
-            if (
-                _CURRENT_PROCESS.pid == current_pid
-                and _CURRENT_PROCESS.machine_id == machine.id
-                and timezone.now() < _CURRENT_PROCESS.modified_at + timedelta(seconds=PROCESS_RECHECK_INTERVAL)
-            ):
-                if _CURRENT_PROCESS.iface_id != iface.id:
-                    _CURRENT_PROCESS.iface = iface
-                    _CURRENT_PROCESS.save(update_fields=["iface", "modified_at"])
-                _CURRENT_PROCESS.ensure_log_files()
-                return _CURRENT_PROCESS
-            _CURRENT_PROCESS = None
-
         # Get actual process start time from OS for validation
         os_start_time = None
         if PSUTIL_AVAILABLE:
@@ -1660,9 +1628,8 @@ class Process(ModelWithDeleteAfter, models.Model):
         if machine is not None:
             stale = stale.filter(machine=machine)
 
-        # Recovery can run against damaged DB state; stream rows so a large
-        # stale Process backlog cannot be materialized in memory at once.
-        for proc in stale.iterator(chunk_size=100):
+        # Close each bounded read before inspecting processes or writing their state.
+        for proc in stale.order_by("pk").paged_iterator(chunk_size=100):
             shares_pid_namespace = proc.shares_pid_namespace
             if shares_pid_namespace and proc.poll() is not None:
                 cleaned += 1
@@ -1778,7 +1745,8 @@ class Process(ModelWithDeleteAfter, models.Model):
             return None
 
         # Can't validate processes on other machines
-        if self.machine_id != Machine.current().id:
+        machine = Machine.current_readonly()
+        if machine is None or self.machine_id != machine.id:
             return None
 
         try:
@@ -2601,14 +2569,17 @@ class Process(ModelWithDeleteAfter, models.Model):
         """
         cleaned = 0
 
+        # Select active IDs through the existing status index first. Combining
+        # type/status directly let SQLite prefer the historical type index and
+        # scan 350k processes to return zero workers (blocking writes for 12s).
+        active_ids = cls.objects.filter(status=cls.StatusChoices.RUNNING).order_by().values("pk")
         running_children = cls.objects.filter(
+            pk__in=active_ids,
             process_type__in=[cls.TypeChoices.WORKER, cls.TypeChoices.HOOK],
-            status=cls.StatusChoices.RUNNING,
         )
 
-        # Recovery can run against damaged DB state; stream rows so a large
-        # orphaned Process backlog cannot be materialized in memory at once.
-        for proc in running_children.iterator(chunk_size=100):
+        # Close each bounded read before inspecting processes or writing their state.
+        for proc in running_children.order_by("pk").paged_iterator(chunk_size=100):
             if not proc.is_running:
                 proc.mark_exited(
                     exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),

archivebox/misc/checks.py

@@ -194,7 +194,10 @@ def check_not_root():
     if os.geteuid() == 0 and not (is_getting_help or is_getting_version):
         print("[yellow][!] Running ArchiveBox as root is not recommended.[/yellow]", file=sys.stderr)
         print("    Root-owned DATA_DIR files may be inaccessible to non-root users later.", file=sys.stderr)
-        print("        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root", file=sys.stderr)
+        print(
+            "        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user",
+            file=sys.stderr,
+        )
 
 
 def is_archivebox_source_root(path: Path | str | None = None) -> bool:
@@ -255,7 +258,7 @@ def check_data_dir_permissions(config=None, **config_kwargs):
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions]https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions[/link]",
         )
         STDERR.print(
-            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid[/link]",
+            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues[/link]",
         )
         STDERR.print(
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts[/link]",

archivebox/misc/db.py

@@ -240,89 +240,38 @@ def drop_models_on_postgres(apps, schema_editor, app_label: str, model_names: li
             schema_editor.delete_model(model)
 
 
-def run_db_analyze_batch(
-    remaining: list[str] | None,
-    *,
-    max_seconds_per_table: float = 120.0,
-) -> list[str]:
-    """Advance one step of a batched SQLite ``ANALYZE`` sweep.
+def run_db_analyze_batch(remaining: list[str] | None) -> list[str]:
+    """Refresh approximate SQLite planner stats for one table per idle tick.
 
-    Without periodic ANALYZE the optimizer's table stats go stale as
-    snapshot/archiveresult tables grow, causing it to start large joins from
-    ``auth_user`` instead of using the indexed url column and blowing snapshot
-    detail page render time from ~50ms to ~500ms+.
-
-    The whole sweep is spread across many calls instead of running as one
-    blocking ``ANALYZE``: pass ``None`` to start a fresh sweep (this call
-    enumerates user tables and runs ``ANALYZE`` on the first one); pass the
-    returned list to advance one more table on each subsequent call. An
-    empty return value means the sweep is complete (or has been aborted) and
-    the next caller should pass ``None`` again. Caller is responsible for
-    throttling new sweeps (orchestrator starts at most one per 24hr while
-    idle) and enforcing a hard upper bound on total sweep wall time.
-
-    Safety guarantees:
-
-    - **Never raises**: every database call is wrapped; on any failure the
-      function returns ``[]`` (abandoning the rest of the sweep) so the
-      orchestrator never crashes on maintenance errors.
-    - **Bounded per-call wall time**: a SQLite progress handler aborts the
-      current ``ANALYZE`` statement once ``max_seconds_per_table`` is
-      exceeded, so a single pathological table cannot wedge the call.
-    - **Never leaves the db locked**: each ``ANALYZE`` runs as a single
-      statement transaction that auto-commits (or rolls back on
-      abort/error). The cursor and progress handler are always cleaned up
-      in ``finally`` blocks even if Python raises mid-call.
-    - Silent no-op on non-SQLite backends.
-
-    WAL journal mode (set in Django settings) keeps readers fully unblocked
-    throughout; the writer lock is only held for the brief ``sqlite_stat*``
-    flush after each table completes.
+    A full ANALYZE scans every index while holding a write transaction; moving
+    it into the runner did not prevent it from blocking submissions. Sampling
+    at most about 1000 entries per index keeps maintenance independent of
+    collection size. Approximate stats are sufficient for choosing indexes.
+    PostgreSQL owns its statistics through autovacuum/autoanalyze.
     """
     from django.db import connection
 
     if connection.vendor != "sqlite":
         return []
-
-    if remaining is None:
-        try:
-            with connection.cursor() as cursor:
-                cursor.execute(
-                    "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
-                )
-                remaining = [row[0] for row in cursor.fetchall()]
-        except Exception:
-            return []
-
-    if not remaining:
-        return []
-
-    next_table, *rest = remaining
-    raw_conn = connection.connection
-    progress_handler_set = False
-    if raw_conn is not None and max_seconds_per_table > 0:
-        deadline = time.monotonic() + max_seconds_per_table
-        try:
-            raw_conn.set_progress_handler(lambda: 1 if time.monotonic() > deadline else 0, 10000)
-            progress_handler_set = True
-        except Exception:
-            progress_handler_set = False
-
     try:
         with connection.cursor() as cursor:
-            cursor.execute(f'ANALYZE "{next_table}"')
-    except Exception:
-        # Aborted by progress handler, locked db, or any other failure — skip
-        # this table and continue the sweep. ANALYZE is idempotent so we can
-        # retry on the next 24hr sweep.
-        pass
-    finally:
-        if progress_handler_set and raw_conn is not None:
+            if remaining is None:
+                cursor.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")
+                remaining = [row[0] for row in cursor.fetchall()]
+            if not remaining:
+                return []
+            next_table, *rest = remaining
+            cursor.execute("PRAGMA analysis_limit")
+            previous_limit = int(cursor.fetchone()[0])
             try:
-                raw_conn.set_progress_handler(None, 0)
-            except Exception:
-                pass
-    return rest
+                cursor.execute("PRAGMA analysis_limit = 1000")
+                cursor.execute(f"ANALYZE {connection.ops.quote_name(next_table)}")
+            finally:
+                cursor.execute(f"PRAGMA analysis_limit = {previous_limit}")
+            return rest
+    except Exception:
+        # Optional planner maintenance must not prevent queued captures.
+        return []
 
 
 def compact_command(cmdline: list[str] | None, fallback: str = "") -> str:
@@ -526,6 +475,8 @@ HISTORICAL_GHOST_MIGRATIONS: frozenset[tuple[str, str]] = frozenset(
         ("core", "0053_remove_snapshottag_snapshot_old"),
         ("core", "0054_alter_snapshot_timestamp"),
         ("core", "0055_alter_tag_slug"),
+        # Dev-only progress indexes removed in favor of existing-index queries.
+        ("core", "0056_progress_covering_indexes"),
         ("core", "0056_remove_tag_uuid"),
         ("core", "0057_rename_id_tag_old_id"),
         ("core", "0058_alter_tag_old_id"),

archivebox/misc/jsonl.py

@@ -142,3 +142,27 @@ def write_record(record: dict[str, Any], stream: TextIO | None = None) -> None:
     active_stream: TextIO = sys.stdout if stream is None else stream
     active_stream.write(json.dumps(record) + "\n")
     active_stream.flush()
+
+
+def write_records(records: Iterable[dict[str, Any]], stream: TextIO | None = None) -> int:
+    """Stream bulk JSONL output with bounded buffering, returning its row count."""
+    active_stream: TextIO = sys.stdout if stream is None else stream
+    lines: list[str] = []
+    pending_size = 0
+    count = 0
+    for record in records:
+        line = json.dumps(record) + "\n"
+        lines.append(line)
+        pending_size += len(line)
+        count += 1
+        # CLI stdout can be unbuffered. A syscall per snapshot makes large
+        # exports IO-bound; small batches keep memory and pipe latency bounded.
+        if pending_size >= 64 * 1024:
+            active_stream.write("".join(lines))
+            active_stream.flush()
+            lines.clear()
+            pending_size = 0
+    if lines:
+        active_stream.write("".join(lines))
+        active_stream.flush()
+    return count

archivebox/misc/serve_static.py

@@ -32,6 +32,7 @@ from archivebox.misc.logging_util import printable_filesize
 from archivebox.plugins.discovery import render_plugin_full_response
 
 _HASHES_CACHE: dict[Path, tuple[float, dict[str, str]]] = {}
+DIRECTORY_PREVIEW_MAX_BYTES = 100 * 1024
 FAVICON_CACHE_CONTROL = "public, max-age=31536000, s-maxage=31536000, immutable"
 IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\s(?:src|data-src)=["\'])([^"\']+)(["\'])', re.IGNORECASE)
 TRANSFORMED_HTML_PREVIEW_STYLE = """<style id="archivebox-static-html-preview-style">
@@ -69,6 +70,192 @@ a > img:not([width]):not([height]) {
 </style>"""
 
 
+def add_replay_image_fallback(response: HttpResponse) -> HttpResponse:
+    """Add a best-effort responsive-image fallback to the locally served replay worker.
+
+    WHY: a recorder saves the image Chrome selected, not every srcset candidate.
+    Replaying at another viewport/DPR can select an uncaptured URL (the motivating
+    case was Drive's missing 2x logo with its 1x logo already in the same WACZ).
+    WACZ requests are answered inside Webrecorder's service worker, so a Django
+    404 handler cannot repair them. Append this adapter when serving that worker;
+    do not change captures, request more live assets, or resize the recorded page.
+
+    Only an image GET that already returned 404 is eligible. The worker reads the
+    referring HTML from the SAME archive and tries src/srcset candidates declared
+    on that img or its enclosing picture. It never guesses filename substitutions
+    or borrows an unrelated nearby image. The original response wins on absent
+    metadata, unsupported worker APIs, parse errors, timeout, or failed candidates.
+    This deliberately does not promise to fix JS-only srcsets, CSS backgrounds,
+    corrupt images returning 200, or pages whose archived HTML is too large.
+
+    Keep this self-contained in the static-file server: it is a replay repair,
+    independent of the capturing plugin/site. Feature detection makes upstream
+    worker changes disable the repair rather than break ordinary replay. The
+    archive's HTML/ZIP bytes and successful HTTP responses remain untouched.
+    """
+    if (
+        response.status_code != 200
+        or response.streaming
+        or not response.has_header("Service-Worker-Allowed")
+        or "javascript" not in response.get("Content-Type", "")
+    ):
+        return response
+    script = rb"""
+;(() => {
+  try {
+    const replay = self.sw;
+    if (!replay || typeof replay.handleFetch !== "function" ||
+        typeof replay.getResponseFor !== "function" || !replay.collections ||
+        replay.proxyOriginMode || replay.topFramePassthrough || replay.__abxImageFallback) return;
+    const original = replay.handleFetch;
+    const documents = new Map();
+    const decode = value => value.replace(/&(?:amp|quot|apos|lt|gt|#\d+|#x[\da-f]+);/gi, entity => {
+      const named = {"&amp;":"&", "&quot;":'"', "&apos;":"'", "&lt;":"<", "&gt;":">"};
+      return named[entity.toLowerCase()] || String.fromCodePoint(
+        entity[2].toLowerCase() === "x" ? parseInt(entity.slice(3), 16) : parseInt(entity.slice(2), 10));
+    });
+    const attributes = tag => {
+      const attrs = {};
+      for (const m of tag.matchAll(/([^\s=/>]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+))/g)) {
+        attrs[m[1].toLowerCase()] = decode(m[2] ?? m[3] ?? m[4]);
+      }
+      return attrs;
+    };
+    const sourceSet = value => {
+      // URLs may contain commas (e.g. image-CDN transforms). Like HTML's
+      // srcset tokenizer, consume a URL through whitespace, then descriptors.
+      const urls = [];
+      let rest = value || "";
+      while (rest && urls.length < 64) {
+        rest = rest.replace(/^[\s,]+/, "");
+        const token = rest.match(/^\S+/)?.[0];
+        if (!token) break;
+        urls.push(token.replace(/,+$/, ""));
+        rest = rest.slice(token.length);
+        if (!token.endsWith(",")) rest = rest.replace(/^[^,]*(?:,|$)/, "");
+      }
+      return urls;
+    };
+    const splitReplay = (url, prefix) => {
+      if (!url.startsWith(prefix)) return null;
+      return url.slice(prefix.length).match(/^((?::[^/]+\/)?\d{0,14})(?:[a-z]+_)?\/((?:https?:)?\/\/.+)$/);
+    };
+    replay.handleFetch = async function(event) {
+      // Never catch/retry the original operation: preserve its normal behavior.
+      const response = await original.call(this, event);
+      const request = event.request;
+      if (response.status !== 404 || request.method !== "GET" || request.destination !== "image") return response;
+      let expired = false, timer;
+      try {
+        const repair = async () => {
+          if (!request.url.startsWith(this.replayPrefix)) return response;
+          const name = this.collections.root || request.url.slice(this.replayPrefix.length).split("/", 1)[0];
+          const collection = await this.collections.getColl(name);
+          // Do not operate on live/proxy collections or remote source archives.
+          // lookupUrl is the archive index API, absent on the live proxy store.
+          if (!collection || typeof collection.store?.lookupUrl !== "function" || collection.liveRedirectOnNotFound ||
+              new URL(collection.config.sourceUrl).origin !== self.location.origin) return response;
+          const missing = splitReplay(request.url, collection.prefix);
+          const referring = splitReplay(request.referrer, collection.prefix);
+          if (!missing || !referring) return response;
+          const archiveURL = url => collection.prefix + referring[1] + "id_/" + url;
+          const read = url => this.getResponseFor(new Request(archiveURL(url), {
+            credentials: "same-origin", redirect: "manual", signal: request.signal,
+          }), event);
+          const key = archiveURL(referring[2]);
+          if (!documents.has(key)) {
+            // Cache only candidate metadata, never whole HTML or image bodies.
+            // Both entry count and HTML reads are bounded; WACZ range loading
+            // remains the upstream replay engine's responsibility.
+            if (documents.size >= 16) documents.delete(documents.keys().next().value);
+            documents.set(key, (async () => {
+              const html = await read(referring[2]);
+              if (html.status !== 200 || !html.headers.get("content-type")?.includes("text/html")) return [];
+              const reader = html.body.getReader(), decoder = new TextDecoder();
+              let text = "", size = 0;
+              try {
+                while (true) {
+                  const chunk = await reader.read();
+                  if (chunk.done) break;
+                  size += chunk.value.byteLength;
+                  if (expired || size > 2 * 1024 * 1024) return [];
+                  text += decoder.decode(chunk.value, {stream:true});
+                }
+                text += decoder.decode();
+              } finally {await reader.cancel().catch(() => {});}
+              // Ignore tag-shaped strings in scripts/comments. Parsing is
+              // intentionally conservative; an odd/malformed page stays broken
+              // instead of acquiring an unrelated "similar looking" image.
+              text = text.replace(/<!--[\s\S]*?-->|<(script|style)\b[^>]*>[\s\S]*?<\/\1\s*>/gi, "");
+              const groups = [];
+              let base = referring[2], picture = null;
+              const absolute = value => {
+                try {
+                  const url = new URL(value, base);
+                  if (!/^https?:$/.test(url.protocol) || url.username || url.password) return null;
+                  url.hash = "";
+                  return url.href;
+                } catch {return null;}
+              };
+              for (const token of text.matchAll(/<\/?(?:base|picture|source|img)\b(?:"[^"]*"|'[^']*'|[^'">])*>/gi)) {
+                const tag = token[0], attrs = attributes(tag);
+                if (/^<base\b/i.test(tag)) {if (attrs.href) base = absolute(attrs.href) || base; continue;}
+                if (/^<picture\b/i.test(tag)) {picture = []; continue;}
+                if (/^<\/picture\b/i.test(tag)) {picture = null; continue;}
+                const urls = [...new Set([attrs.src, ...sourceSet(attrs.srcset)].filter(Boolean).map(absolute).filter(Boolean))];
+                if (/^<source\b/i.test(tag)) {if (picture) picture.push(...urls); continue;}
+                if (!attrs.srcset && !picture?.length) continue;
+                groups.push([...new Set([...urls, ...(picture || [])])].slice(0, 64));
+                if (groups.length >= 512) break;
+              }
+              return groups;
+            })());
+          }
+          const groups = await documents.get(key);
+          const failed = new URL(missing[2], referring[2]); failed.hash = "";
+          const candidates = groups.find(group => group.includes(failed.href));
+          if (!candidates) return response;
+          // Prefer the img's explicit src, then alternatives in declaration
+          // order. Limit extra archive lookups, and never follow redirects.
+          for (const url of candidates.filter(url => url !== failed.href).slice(0, 6)) {
+            if (expired) break;
+            const alternative = await read(url);
+            if (alternative.status !== 200 || !alternative.headers.get("content-type")?.startsWith("image/")) continue;
+            if (expired) break;
+            const headers = new Headers(alternative.headers);
+            headers.set("X-ArchiveBox-Image-Fallback", url);
+            headers.set("Cache-Control", "no-store");
+            return new Response(alternative.body, {status:200, headers});
+          }
+          return response;
+        };
+        return await Promise.race([
+          repair().catch(() => response),
+          new Promise(resolve => {timer = setTimeout(() => {expired = true; resolve(response);}, 1500);}),
+        ]);
+      } catch {return response;}
+      finally {expired = true; clearTimeout(timer);}
+    };
+    replay.__abxImageFallback = true;
+  } catch { /* Optional repair must never prevent the worker from starting. */ }
+})();
+"""
+    try:
+        body = response.content
+        if b"self.sw=" not in body or b"__abxImageFallback" in body:
+            return response
+        response.content = body + script
+        # Upstream headers describe the unmodified worker. Revalidate this
+        # generated asset so existing captures can receive the repair too.
+        for header in ("ETag", "Last-Modified", "Content-Length"):
+            if response.has_header(header):
+                del response[header]
+        response["Cache-Control"] = "no-cache"
+    except Exception:
+        pass
+    return response
+
+
 def _load_hash_map(snapshot_dir: Path) -> dict[str, str] | None:
     hashes_path = snapshot_dir / "hashes" / "hashes.json"
     if not hashes_path.exists():
@@ -126,7 +313,8 @@ def _resolve_archive_path(document_root: str | Path, rel_path: str) -> tuple[Pat
         current = match
         resolved_parts.append(match.name)
 
-    return current, posixpath.join(*resolved_parts) if resolved_parts else ""
+    resolved_path = posixpath.join(*resolved_parts) if resolved_parts else ""
+    return Path(safe_join(document_root, resolved_path)), resolved_path
 
 
 def _cache_policy(config=None, **config_kwargs) -> str:
@@ -308,10 +496,24 @@ def _render_directory_index(request, path: str, fullpath: Path) -> HttpResponse:
         file_list.append(url)
 
         stat_result = entry.stat()
+        preview_kind = ""
+        mime = mimetypes.guess_type(entry.name)[0] or "application/octet-stream"
+        if entry.is_file() and stat_result.st_size < DIRECTORY_PREVIEW_MAX_BYTES:
+            if mime.startswith("image/"):
+                preview_kind = "image"
+            elif (
+                mime.startswith("text/")
+                or re.search(r"(?:^|[.+/-])(?:json|xml|javascript|ecmascript|yaml|toml|graphql|sql)(?:[.+/-]|$)", mime)
+                or entry.suffix.lower() in {".log", ".jsonl", ".toml"}
+            ):
+                preview_kind = "text"
         entries.append(
             {
                 "name": url,
                 "url": url,
+                "preview_kind": preview_kind,
+                "mimetype": "Directory" if entry.is_dir() else mime,
+                "size_bytes": 0 if entry.is_dir() else stat_result.st_size,
                 "is_dir": entry.is_dir(),
                 "size": "—" if entry.is_dir() else printable_filesize(stat_result.st_size),
                 "timestamp": _format_direntry_timestamp(stat_result),
@@ -851,7 +1053,8 @@ def _is_risky_replay_document(fullpath: Path, content_type: str) -> bool:
     # Unknown archived response paths often have no extension. Sniff a small prefix
     # so one-domain no-JS mode still catches HTML/SVG documents.
     try:
-        head = fullpath.read_bytes()[:4096].decode("utf-8", errors="ignore").lower()
+        with fullpath.open("rb") as source:
+            head = source.read(4096).decode("utf-8", errors="ignore").lower()
     except (OSError, UnicodeDecodeError):
         return False
 
@@ -1020,6 +1223,29 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             config=config,
         )
 
+    # A shared archive explorer for ZIP outputs from any plugin. Raw/range
+    # requests retain the original bytes for downloads and seekable reads.
+    if fullpath.suffix.lower() == ".zip" and request.GET.get("preview") and not request.META.get("HTTP_RANGE"):
+        raw_query = request.GET.copy()
+        raw_query.pop("preview", None)
+        raw_query["raw"] = "1"
+        wrapped = loader.get_template("static/zip_index.html").render(
+            {
+                "title": fullpath.name,
+                "output_path": f"{request.path}?{raw_query.urlencode()}",
+                "archive_size": statobj.st_size,
+            },
+        )
+        response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
+        _set_transformed_response_headers(response, fullpath, statobj, None, cache_policy)
+        return _apply_archive_replay_headers(
+            response,
+            fullpath=fullpath,
+            content_type="text/html; charset=utf-8",
+            is_archive_replay=is_archive_replay,
+            config=config,
+        )
+
     # Wrap text-like outputs in HTML when explicitly requested for iframe previewing.
     if preview_as_text_html:
         try:

archivebox/misc/shell_welcome_message.py

@@ -48,10 +48,10 @@ if __name__ == "__main__":
 
     prnt("[i] :heavy_dollar_sign: Welcome to the ArchiveBox Shell!")
     prnt(
-        "    [deep_sky_blue4]Docs:[/deep_sky_blue4] [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage[/link]",
+        "    [deep_sky_blue4]Docs:[/deep_sky_blue4] [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage[/link]",
     )
     prnt(
-        "          [link=https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html]https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html[/link]",
+        "          [link=https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html]https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html[/link]",
     )
     prnt()
     prnt(" :grey_question: [violet]Hint[/] [i]Here are some examples to get started:[/]")

archivebox/misc/util.py

@@ -3,7 +3,7 @@ __package__ = "archivebox.misc"
 # Bootable utility functions (URL parsing, date parsing, JSON encoding, decorators).
 # MUST NOT import archivebox.config, archivebox.core, or Django — this module is
 # loaded by hooks.py and other early code paths. Only depends on stdlib and the
-# bootable .logging module.
+# bootable .logging module and standalone abx_plugins URL helpers.
 
 import re
 import json as pyjson
@@ -19,6 +19,9 @@ from urllib.parse import urlparse, quote, unquote
 from html import escape, unescape
 from datetime import datetime, timezone
 
+from abx_plugins.plugins.base.url_cleaning import URL_REGEX, fix_url_from_markdown
+from abx_plugins.plugins.base.url_cleaning import sanitize_extracted_url as sanitize_plugin_url
+
 from .logging import COLOR_DICT
 
 
@@ -90,18 +93,6 @@ def ts_to_date_str(ts: Any) -> str | None:
 COLOR_REGEX = re.compile(r"\[(?P<arg_1>\d+)(;(?P<arg_2>\d+)(;(?P<arg_3>\d+))?)?m")
 
 
-# https://mathiasbynens.be/demo/url-regex
-URL_REGEX = re.compile(
-    r"(?=("
-    r"http[s]?://"  # start matching from allowed schemes
-    r"(?:[a-zA-Z]|[0-9]"  # followed by allowed alphanum characters
-    r"|[-_$@.&+!*\(\),]"  #   or allowed symbols (keep hyphen first to match literal hyphen)
-    r"|[^\u0000-\u007F])+"  #   or allowed unicode bytes
-    r'[^\]\[<>"\'\s]+'  # stop parsing at these symbols
-    r"))",
-    re.IGNORECASE | re.UNICODE,
-)
-
 # Maximum supported URL length. Very long URLs are rare but must be supported correctly
 # (e.g. data: URLs, deeply nested query strings). The Snapshot.url column is stored as a
 # variable-length TextField (so short URLs don't reserve space and very long URLs still
@@ -109,29 +100,6 @@ URL_REGEX = re.compile(
 # all keep working.
 MAX_URL_LENGTH = 65535
 
-QUOTE_DELIMITERS = (
-    '"',
-    "'",
-    "`",
-    "“",
-    "”",
-    "‘",
-    "’",
-)
-QUOTE_ENTITY_DELIMITERS = (
-    "&quot;",
-    "&#34;",
-    "&#x22;",
-    "&apos;",
-    "&#39;",
-    "&#x27;",
-)
-URL_ENTITY_REPLACEMENTS = (
-    ("&amp;", "&"),
-    ("&#38;", "&"),
-    ("&#x26;", "&"),
-)
-
 FILESIZE_UNITS: dict[str, int] = {
     "": 1,
     "b": 1,
@@ -153,36 +121,9 @@ FILESIZE_UNITS: dict[str, int] = {
 
 
 def sanitize_extracted_url(url: str) -> str:
-    """Trim quote garbage and dangling prose punctuation from an extracted URL candidate."""
-    cleaned = (url or "").strip()
-    if not cleaned:
-        return cleaned
-
-    lower_cleaned = cleaned.lower()
-    cut_index = len(cleaned)
-
-    for delimiter in QUOTE_DELIMITERS:
-        found_index = cleaned.find(delimiter)
-        if found_index != -1:
-            cut_index = min(cut_index, found_index)
-
-    for delimiter in QUOTE_ENTITY_DELIMITERS:
-        found_index = lower_cleaned.find(delimiter)
-        if found_index != -1:
-            cut_index = min(cut_index, found_index)
-
-    cleaned = cleaned[:cut_index].strip()
-    lower_cleaned = cleaned.lower()
-    for entity, replacement in URL_ENTITY_REPLACEMENTS:
-        while entity in lower_cleaned:
-            entity_index = lower_cleaned.find(entity)
-            cleaned = cleaned[:entity_index] + replacement + cleaned[entity_index + len(entity) :]
-            lower_cleaned = cleaned.lower()
-
-    cleaned = cleaned.rstrip(".,;:!?\\'\"")
-    cleaned = cleaned.rstrip('"')
-
-    return cleaned
+    """Use the standalone parser's quote rules, then trim dangling prose punctuation."""
+    cleaned = sanitize_plugin_url(url)
+    return cleaned.rstrip('.,;:!?\\"')
 
 
 def validate_url_length(url: str) -> str:
@@ -215,39 +156,6 @@ def parens_are_matched(string: str, open_char="(", close_char=")"):
     return count == 0
 
 
-def fix_url_from_markdown(url_str: str) -> str:
-    """
-    cleanup a regex-parsed url that may contain dangling trailing parens from markdown link syntax
-    helpful to fix URLs parsed from markdown e.g.
-      input:  https://wikipedia.org/en/some_article_(Disambiguation).html?abc=def).somemoretext
-      result: https://wikipedia.org/en/some_article_(Disambiguation).html?abc=def
-
-    IMPORTANT ASSUMPTION: valid urls wont have unbalanced or incorrectly nested parentheses
-    e.g. this will fail the user actually wants to ingest a url like 'https://example.com/some_wei)(rd_url'
-         in that case it will return https://example.com/some_wei (truncated up to the first unbalanced paren)
-    This assumption is true 99.9999% of the time, and for the rare edge case the user can use url_list parser.
-    """
-    trimmed_url = url_str
-    if len(trimmed_url) > 2048:
-        return trimmed_url
-
-    # cut off one trailing character at a time
-    # until parens are balanced e.g. /a(b)c).x(y)z -> /a(b)c
-    trim_attempts = 0
-    while trimmed_url and not parens_are_matched(trimmed_url) and trim_attempts < 256:
-        trimmed_url = trimmed_url[:-1]
-        trim_attempts += 1
-
-    if not trimmed_url or not parens_are_matched(trimmed_url):
-        return url_str
-
-    # make sure trimmed url is still valid
-    if any(match == trimmed_url for match in re.findall(URL_REGEX, trimmed_url)):
-        return trimmed_url
-
-    return url_str
-
-
 def split_comma_separated_urls(url: str):
     offset = 0
     while True:
@@ -274,7 +182,8 @@ def find_all_urls(urls_str: str):
         if match.start() in skipped_starts:
             continue
 
-        cleaned_match = sanitize_extracted_url(fix_url_from_markdown(match.group(1)))
+        cleaned_match = fix_url_from_markdown(match.group(1), preceding_text=urls_str[max(0, match.start(1) - 6) : match.start(1)])
+        cleaned_match = sanitize_extracted_url(cleaned_match)
         for offset, url in split_comma_separated_urls(cleaned_match):
             if offset:
                 skipped_starts.add(match.start() + offset)

archivebox/opencode/__init__.py

@@ -1 +0,0 @@
-"""ArchiveBox OpenCode admin integration."""

archivebox/opencode/urls.py

@@ -1,13 +0,0 @@
-from django.urls import path, re_path
-
-from archivebox.opencode.views import agent_view, opencode_proxy_view
-
-
-urlpatterns = [
-    path("", agent_view, name="opencode-agent"),
-    re_path(
-        r"^opencode(?:/(?P<path>.*))?$",
-        opencode_proxy_view,
-        name="opencode-proxy",
-    ),
-]

archivebox/opencode/views.py

@@ -1,140 +0,0 @@
-"""Django adapter for the optional plugin; never import its runtime at startup."""
-
-import logging
-from io import BytesIO
-
-from asgiref.sync import sync_to_async
-
-from django.contrib.auth.middleware import AuthenticationMiddleware
-from django.contrib.auth.views import redirect_to_login
-from django.contrib.sessions.middleware import SessionMiddleware
-from django.core.exceptions import PermissionDenied
-from django.core.handlers.asgi import ASGIRequest
-from django.db import close_old_connections
-from django.http import Http404, HttpResponse, HttpResponseForbidden, StreamingHttpResponse
-from django.template import engines
-from django.urls import Resolver404, resolve
-from django.views.decorators.csrf import csrf_exempt
-
-from archivebox.config import CONSTANTS
-from archivebox.config.common import get_request_config
-from archivebox.core.middleware import ReverseProxyAuthMiddleware
-from archivebox.core.routes_util import build_admin_url, get_admin_host, get_api_base_url, get_base_url, host_matches
-from archivebox.plugins.discovery import get_plugin_template
-
-_LOGGER = logging.getLogger(__name__)
-
-
-def _runtime_settings(request, config):
-    from abx_plugins.plugins.opencode import runtime
-
-    settings = runtime._settings(config, CONSTANTS.DATA_DIR)
-    route_config = request.__dict__.get("archivebox_config")
-    settings.update(
-        archivebox_base_url=get_base_url(request=request, config=route_config).rstrip("/"),
-        archivebox_admin_url=build_admin_url("/admin/", request=request, config=route_config).rstrip("/"),
-        archivebox_api_url=f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/",
-    )
-    return runtime, settings
-
-
-def _dispatch(request, path=None):
-    try:
-        # Middleware already merged server defaults and live Machine overrides.
-        # Re-resolving every extractor's config here stalls the UI request burst.
-        config = get_request_config(request).model_dump(mode="json")
-        if not config.get("OPENCODE_ENABLED", False):
-            raise Http404
-        if not request.user.is_authenticated:
-            return redirect_to_login(request.get_full_path(), login_url="/admin/login/")
-        if not request.user.is_active or not request.user.is_superuser:
-            return HttpResponseForbidden("Agent access requires a superuser account.")
-
-        runtime, settings = _runtime_settings(request, config)
-        if path is None:
-            from archivebox.core.admin_site import archivebox_admin
-
-            context = {**archivebox_admin.each_context(request), **runtime.agent_context(settings)}
-            source = get_plugin_template("opencode", "agent", fallback=False)
-            if source is None:
-                raise RuntimeError("Agent template unavailable")
-            return HttpResponse(engines["django"].from_string(source).render(context, request))
-
-        if not runtime._origin_allowed(request.method, request.get_host(), request.headers):
-            return HttpResponseForbidden("Cross-origin agent requests are blocked.")
-        status, headers, body = runtime.proxy(
-            settings,
-            request.method,
-            path,
-            tuple((key, value) for key, values in request.GET.lists() for value in values),
-            request.headers,
-            request.body,
-        )
-        response_type = HttpResponse if isinstance(body, bytes) else StreamingHttpResponse
-        response = response_type(body, status=status, headers=headers)
-        response.xframe_options_exempt = True
-        response.headers["X-Frame-Options"] = "SAMEORIGIN"
-        response.headers["Content-Security-Policy"] = "frame-ancestors 'self'"
-        return response
-    except Http404:
-        raise
-    except Exception:
-        # Optional-service boundary, including imports and template rendering.
-        _LOGGER.exception("Optional AI service failed")
-        return HttpResponse("AI service unavailable. See server logs.", status=503, content_type="text/plain")
-
-
-def agent_view(request):
-    return _dispatch(request)
-
-
-@csrf_exempt
-def opencode_proxy_view(request, path=None):
-    return _dispatch(request, path=path or "")
-
-
-def _websocket_context(scope):
-    close_old_connections()
-    try:
-        match = resolve(scope["path"])
-        if match.url_name != "opencode-proxy":
-            raise PermissionDenied
-        request = ASGIRequest(
-            {**scope, "type": "http", "method": "GET", "scheme": "https" if scope.get("scheme") == "wss" else "http"},
-            BytesIO(),
-        )
-        route_config = get_request_config(request, resolve_plugins=False)
-        request.archivebox_config = route_config
-        if not route_config.CONTROL_PLANE_ENABLED:
-            raise PermissionDenied
-        if (
-            route_config.USES_SUBDOMAIN_ROUTING
-            and route_config.BASE_URL
-            and not host_matches(request.get_host(), get_admin_host(config=route_config, request=request))
-        ):
-            raise PermissionDenied
-        SessionMiddleware(_dispatch).process_request(request)
-        AuthenticationMiddleware(_dispatch).process_request(request)
-        ReverseProxyAuthMiddleware(_dispatch).process_request(request)
-        config = route_config.model_dump(mode="json")
-        if not config.get("OPENCODE_ENABLED") or not request.user.is_active or not request.user.is_superuser:
-            raise PermissionDenied
-        runtime, settings = _runtime_settings(request, config)
-        if not request.headers.get("Origin") or not runtime._origin_allowed("POST", request.get_host(), request.headers):
-            raise PermissionDenied
-        return runtime, settings, match.kwargs.get("path", "")
-    finally:
-        close_old_connections()
-
-
-async def websocket_view(scope, receive, send):
-    if (await receive())["type"] != "websocket.connect":
-        return
-    try:
-        runtime, settings, path = await sync_to_async(_websocket_context)(scope)
-        await runtime.websocket_proxy(settings, path, scope.get("query_string", b""), scope.get("subprotocols", []), receive, send)
-    except (PermissionDenied, Resolver404, Http404):
-        await send({"type": "websocket.close", "code": 1008})
-    except Exception:
-        _LOGGER.exception("Optional AI WebSocket failed")
-        await send({"type": "websocket.close", "code": 1011})

archivebox/personas/forms.py

@@ -122,14 +122,14 @@ class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm):
                 "Use the custom path/CDP option if the browser data lives elsewhere."
             )
 
-        self.build_plugin_groups(get_config(persona=self.instance) if self.instance and self.instance.pk else get_config())
+        self.build_plugin_groups(get_config(persona=self.instance) if not self.instance._state.adding else get_config())
 
     def clean_name(self) -> str:
         name = str(self.cleaned_data.get("name") or "").strip()
         is_valid, error_message = validate_persona_name(name)
         if not is_valid:
             raise forms.ValidationError(error_message)
-        return name
+        return self.instance.validate_name(name)
 
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()

archivebox/personas/models.py

@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any
 from collections.abc import Mapping
 
 from django.db import IntegrityError, models
+from django.core.exceptions import ValidationError
 from django.db.models.fields.json import KT
 from django.conf import settings
 from django.utils import timezone
@@ -74,8 +75,12 @@ def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, person
         derived["COOKIES_FILE"] = str(cookies_path)
 
     auth_path = persona_dir / "auth.json"
-    if "AUTH_STORAGE_FILE" not in derived and auth_path.exists():
-        derived["AUTH_STORAGE_FILE"] = str(auth_path)
+    if "AUTH_STORAGE_FILE" not in derived:
+        try:
+            if auth_path.stat().st_size > 0:
+                derived["AUTH_STORAGE_FILE"] = str(auth_path)
+        except FileNotFoundError:
+            pass
 
     derived["ACTIVE_PERSONA"] = name
     return derived
@@ -117,6 +122,13 @@ class Persona(ModelWithConfig):
         app_label = "personas"
 
     def save(self, *args, **kwargs):
+        # Adopt the spelling on disk, never rename a Chrome profile just to fix case.
+        self.__dict__.pop("_persona_path", None)
+        canonical_name = self.validate_name(self.name)
+        if canonical_name != self.name:
+            self.name = canonical_name
+            if kwargs.get("update_fields") is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*kwargs["update_fields"], "name"]))
         config = dict(self.config or {})
         if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
             from archivebox.config.common import get_config
@@ -133,10 +145,59 @@ class Persona(ModelWithConfig):
 
     @property
     def path(self) -> Path:
-        """Path to persona directory under PERSONAS_DIR."""
+        """Reuse the directory's spelling without renaming a Chrome profile."""
         from archivebox.config.constants import CONSTANTS
 
-        return CONSTANTS.PERSONAS_DIR / self.name
+        cached = self.__dict__.get("_persona_path")
+        if cached is None or cached[0] != self.name:
+            directory_name = self.directory_name(self.name)
+            cached = (self.name, CONSTANTS.PERSONAS_DIR / directory_name)
+            self.__dict__["_persona_path"] = cached
+        return cached[1]
+
+    @staticmethod
+    def directory_name(name: str) -> str:
+        from archivebox.config.constants import CONSTANTS
+        from archivebox.personas.importers import validate_persona_name
+
+        valid, error = validate_persona_name(name)
+        if not valid:
+            raise ValidationError(error)
+        # Linux can hold both Default/default; macOS usually aliases them. Reuse a
+        # single match on either filesystem, but never guess which conflicting
+        # profile contains the user's login state. Only list names, not profile trees.
+        try:
+            with os.scandir(CONSTANTS.PERSONAS_DIR) as entries:
+                matches = sorted(entry.name for entry in entries if entry.name.casefold() == name.casefold())
+        except FileNotFoundError:
+            matches = []
+        if len(matches) > 1:
+            raise ValidationError(
+                f"Conflicting persona directories: {'; '.join(str(CONSTANTS.PERSONAS_DIR / match) for match in matches)}. "
+                "Stop Chrome and rename the conflicting directories manually to distinct names; no profile directories were renamed.",
+            )
+        return matches[0] if matches else name
+
+    @classmethod
+    def find_named(cls, name: str) -> "Persona | None":
+        # SQLite's iexact is ASCII-only; Python casefold gives every entry point
+        # the same Unicode matching rules as the filesystem-name lookup above.
+        matches = [(pk, existing) for pk, existing in cls.objects.values_list("pk", "name") if existing.casefold() == name.casefold()]
+        if len(matches) > 1:
+            raise ValidationError(
+                f"Conflicting persona records: {'; '.join(f'{existing!r} (ID {pk})' for pk, existing in matches)}. "
+                "Rename the conflicting personas manually to distinct names; profile files have not been changed.",
+            )
+        return cls.objects.get(pk=matches[0][0]) if matches else None
+
+    def validate_name(self, name: str) -> str:
+        existing = self.find_named(name)
+        if existing is not None and existing.pk != self.pk:
+            raise ValidationError(
+                f"Name conflicts with persona {existing.name!r} (ID {existing.pk}, directory {existing.path}). "
+                "Choose a distinct name or rename that persona record manually.",
+            )
+        return self.directory_name(name)
 
     @property
     def CHROME_USER_DATA_DIR(self) -> str:
@@ -156,9 +217,12 @@ class Persona(ModelWithConfig):
 
     @property
     def AUTH_STORAGE_FILE(self) -> str:
-        """Derived path to auth.json for this persona (if it exists)."""
+        """Derived path to a nonempty auth.json export for this persona."""
         auth_path = self.path / "auth.json"
-        return str(auth_path) if auth_path.exists() else ""
+        try:
+            return str(auth_path) if auth_path.stat().st_size > 0 else ""
+        except FileNotFoundError:
+            return ""
 
     def get_derived_config(self) -> dict:
         """
@@ -171,10 +235,13 @@ class Persona(ModelWithConfig):
         - AUTH_STORAGE_FILE (derived from persona path, if file exists)
         - ACTIVE_PERSONA (set to this persona's name)
         """
-        return derive_persona_config(name=self.name, config=self.config, persona_dir=self.path)
+        return derive_persona_config(name=self.path.name, config=self.config, persona_dir=self.path)
 
     def ensure_dirs(self) -> None:
         """Create persona directories if they don't exist."""
+        # Recheck at profile preparation, including changes made outside this process.
+        self.__dict__.pop("_persona_path", None)
+        self.validate_name(self.name)
         self.path.mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_profile").mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_downloads").mkdir(parents=True, exist_ok=True)
@@ -246,16 +313,26 @@ class Persona(ModelWithConfig):
                     fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
 
     @classmethod
-    def get_or_create_named(cls, name: str) -> "Persona":
+    def get_or_create_named(cls, name: str, *, defaults: dict | None = None) -> "Persona":
+        # Init/startup must reuse a user-created 'default', not create a second
+        # DB row pointing at a different (or shared) profile (issue #1896).
         persona_name = (name or "Default").strip() or "Default"
-        persona = cls.objects.filter(name=persona_name).first()
+        persona = cls.find_named(persona_name)
         if persona is not None:
+            canonical_name = persona.validate_name(persona.name)
+            if canonical_name != persona.name:
+                persona.name = canonical_name
+                persona.save(update_fields=["name"])
             return persona
 
         try:
-            return cls.objects.create(name=persona_name)
-        except IntegrityError:
-            return cls.objects.get(name=persona_name)
+            return cls.objects.create(name=persona_name, **(defaults or {}))
+        except (IntegrityError, ValidationError):
+            persona = cls.find_named(persona_name)
+            if persona is None:
+                raise
+            persona.validate_name(persona.name)
+            return persona
 
     def runtime_root_for_crawl(self, crawl) -> Path:
         return Path(crawl.output_dir) / ".persona" / self.name
@@ -315,20 +392,9 @@ class Persona(ModelWithConfig):
             if chrome_binary:
                 (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # Hooks derive CHROME_USER_DATA_DIR/CHROME_DOWNLOADS_DIR from
-            # PERSONAS_DIR + ACTIVE_PERSONA. Point PERSONAS_DIR at the
-            # per-crawl runtime root here so CHROME_ISOLATION=crawl never
-            # leaks or reuses the template profile while keeping Chrome path
-            # derivation centralized in the Chrome plugin helpers.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        # Hooks derive Chrome profile paths from the runtime persona root.
+        # Use the same optional auth discovery as the template persona.
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
         crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
@@ -358,18 +424,7 @@ class Persona(ModelWithConfig):
         if chrome_binary:
             (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # See prepare_runtime_for_crawl(): snapshot isolation changes the
-            # persona root, not individual CHROME_* config keys, so standalone
-            # Chrome hooks and ArchiveBox-driven hooks resolve paths the same way.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def cleanup_runtime_for_crawl(self, crawl) -> None:
         shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True)

archivebox/plugins/discovery.py

@@ -192,7 +192,9 @@ def serve_plugin_replay_asset(path: str, config, response_class):
     for module in get_plugin_presentation_modules():
         hook = getattr(module, "serve_replay_asset_response", None)
         if hook is not None and (response := hook(path, config, response_class)) is not None:
-            return response
+            from archivebox.misc.serve_static import add_replay_image_fallback
+
+            return add_replay_image_fallback(response)
     return None
 
 

archivebox/progressmonitor/collection.py

@@ -1,4 +1,15 @@
-"""Non-blocking, database-only collection totals for the progress monitor."""
+"""Cached display totals for progress polling and the public snapshot list.
+
+Missing/stale statistics are intentional: returning useful rows and progress
+quickly matters more than exact initial counts. Each viewer may poll once per
+second; repeating aggregates per request would consume the small server's CPU
+and hold SQLite read locks needed by the archiving runner. Callers must accept
+None on a cold cache and must never wait for the refresh or count synchronously.
+
+Background reads still take normal SQLite read locks. Existing indexes, short
+batches where needed, and throttled refreshes limit their cost; moving work to
+a thread does not make a scan free or require switching the database to WAL.
+"""
 
 import logging
 from threading import Lock, Thread
@@ -9,14 +20,77 @@ from django.db import close_old_connections, connections
 from django.db.models import Count, Sum
 
 
+# This is only a process-local gate for background work, not a DB/file lock.
+# Every request tries it without waiting; a busy refresh means returning the
+# cached value (or None). Serializing refreshes also avoids competing scans for
+# different crawls/users when many viewer tabs open at once.
 _refresh_lock = Lock()
 
 
-def collection_summary(user):
-    user_id = None if user.is_superuser else user.pk
-    key = f"progress-collection:{'all' if user_id is None else user_id}"
+def crawl_summary(crawl_id, *, snapshot_id=None):
+    """Sample sealed snapshot metadata without adding table scans to polls."""
+    key = f"progress-crawl:{crawl_id}:{snapshot_id or 'all'}"
     summary = cache.get(key)
-    # Even an indexed SUM scans the index. Never make an HTTP request wait for
+    if (summary is None or time() - summary["sampled_at"] >= 30) and _refresh_lock.acquire(blocking=False):
+        if not cache.add(f"{key}:refresh", True, timeout=30):
+            _refresh_lock.release()
+            return summary
+
+        def refresh():
+            try:
+                close_old_connections()
+                from archivebox.core.models import Snapshot
+
+                snapshots = Snapshot.objects.filter(crawl_id=crawl_id, status=Snapshot.StatusChoices.SEALED)
+                if snapshot_id is not None:
+                    snapshots = snapshots.filter(pk=snapshot_id)
+                snapshots = snapshots.order_by("modified_at").values_list("modified_at", "downloaded_at", "output_size")
+                totals = {"snapshots": 0, "cancelled": 0, "bytes": 0}
+                modified_at, offset = None, 0
+                while True:
+                    batch_scope = snapshots.filter(modified_at__gte=modified_at) if modified_at is not None else snapshots
+                    # Use (crawl, status, modified_at), releasing the SQLite read
+                    # cursor after each small batch. Never hold a cursor over the
+                    # whole crawl, even in this background worker.
+                    batch = list(batch_scope[offset : offset + 100])
+                    for _, downloaded_at, size in batch:
+                        totals["snapshots"] += 1
+                        totals["cancelled"] += downloaded_at is None
+                        totals["bytes"] += size or 0
+                    if len(batch) < 100:
+                        break
+                    last_modified_at = batch[-1][0]
+                    # Only offset within ties; otherwise advance the indexed
+                    # timestamp range instead of rescanning previous pages.
+                    offset = (offset if last_modified_at == modified_at else 0) + sum(row[0] == last_modified_at for row in batch)
+                    modified_at = last_modified_at
+                totals["sampled_at"] = time()
+                cache.set(key, totals, timeout=3600)
+            except Exception:
+                logging.getLogger(__name__).exception("Could not refresh crawl totals")
+            finally:
+                connections.close_all()
+                _refresh_lock.release()
+
+        try:
+            Thread(target=refresh, name="crawl-summary", daemon=True).start()
+        except RuntimeError:
+            _refresh_lock.release()
+            logging.getLogger(__name__).exception("Could not start crawl totals refresh")
+    return summary
+
+
+def collection_summary(user, *, public=False):
+    """Return cached totals immediately and, when due, start one refresh.
+
+    Public pages need no extra AJAX/polling for these totals: an ordinary later
+    page load reads the populated cache. Keep this best-effort display cache
+    separate from pagination/authorization decisions, which need actual rows.
+    """
+    user_id = None if user.is_superuser else user.pk
+    key = "progress-collection:public" if public else f"progress-collection:{'all' if user_id is None else user_id}"
+    summary = cache.get(key)
+    # Even indexed COUNT/SUM aggregates scan the index. Never make HTTP wait for
     # that scan on a large collection. Serve stale totals while one worker refreshes;
     # a single non-blocking lock prevents a queue of scans from concurrent clients.
     if (summary is None or time() - summary["sampled_at"] >= 60) and _refresh_lock.acquire(blocking=False):
@@ -32,13 +106,29 @@ def collection_summary(user):
                 from archivebox.core.models import Snapshot
 
                 snapshots = Snapshot.objects.all()
-                if user_id is not None:
-                    snapshots = snapshots.filter(crawl__created_by_id=user_id)
-                # COUNT(*) + SUM(output_size) can use Snapshot's covering size index.
-                # Snapshot.output_size is maintained when ArchiveResult sizes change.
-                totals = snapshots.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
-                totals["bytes"] = totals["bytes"] or 0
+                if public:
+                    from archivebox.core.permissions import PERMISSIONS_PUBLIC
+
+                    # SQLite can choose a table scan for equality after ANALYZE
+                    # on an all-public archive. Equal range bounds keep the count
+                    # on the existing permissions index. It is still O(n), so
+                    # public pages share this background cache, never await it.
+                    # Count the indexed, non-null permissions column so this
+                    # needs no Snapshot table reads or new covering-index migration.
+                    totals = snapshots.filter(permissions__gte=PERMISSIONS_PUBLIC, permissions__lte=PERMISSIONS_PUBLIC).aggregate(
+                        snapshots=Count("permissions"),
+                    )
+                else:
+                    if user_id is not None:
+                        snapshots = snapshots.filter(crawl__created_by_id=user_id)
+                    # COUNT(*) + SUM(output_size) can use Snapshot's covering size index.
+                    # Snapshot.output_size is maintained when ArchiveResult sizes change.
+                    totals = snapshots.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
+                totals["bytes"] = totals.get("bytes") or 0
                 totals["sampled_at"] = time()
+                # Retain old totals longer than the 60s refresh interval so a
+                # slow/failed refresh does not force every tab back to a cold
+                # count. Staleness is acceptable; blocking page loads is not.
                 cache.set(key, totals, timeout=3600)
             except Exception:
                 logging.getLogger(__name__).exception("Could not refresh collection totals")

archivebox/progressmonitor/views.py

@@ -7,12 +7,11 @@ from typing import Literal
 from abx_dl.events import PROCESS_EXIT_SKIPPED
 from django.conf import settings
 from django.db import DatabaseError
-from django.db.models import CharField, Count, Q, Sum
+from django.db.models import CharField, Count, Q
 from django.db.models.functions import Cast
 from django.http import HttpResponse, JsonResponse
 from django.utils import timezone
 
-from archivebox.config.common import get_config
 from archivebox.core.permissions import can_view_snapshot, is_admin_user
 from archivebox.core.routes_util import build_snapshot_detail_url, build_snapshot_url, get_api_base_url
 from archivebox.misc.logging_util import printable_filesize
@@ -182,53 +181,37 @@ def live_progress_view(request, *, authorized_snapshot=None):
 
         running_screencasts = {}
 
-        def screencast_frame_url(crawl_id: str, crawl_dir: Path) -> str:
-            frame_path = crawl_dir / "chrome_screencast" / "latest.jpg"
-            try:
-                frame_stat = frame_path.stat()
-            except OSError:
-                return ""
-            if frame_stat.st_size <= 0:
-                return ""
+        def screencast_frame_url(crawl_id: str) -> str:
             started_at = running_screencasts.get(crawl_id)
-            if started_at is None or frame_stat.st_mtime < started_at.timestamp():
+            if started_at is None:
                 return ""
-            rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}"
+            # File existence/freshness belongs to the image request, never to
+            # the one-second JSON poll (archive storage may be remote).
+            rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={int(now.timestamp())}&after={started_at.timestamp()}"
             return f"{api_base}{rel}" if api_base else rel
 
-        machine_id = Machine.current().id
+        machine = Machine.current_readonly()
+        machine_id = machine.id if machine is not None else None
         orchestrator_proc = (
             Process.objects.filter(
                 machine_id=machine_id,
                 process_type=Process.TypeChoices.ORCHESTRATOR,
                 status=Process.StatusChoices.RUNNING,
             )
-            .only("id", "pid", "started_at", "machine_id", "process_type", "status")
+            .only("id", "pid", "started_at", "machine_id", "process_type", "status", "env", "cmd")
             .order_by("-started_at")
             .first()
             if machine_id is not None
             else None
         )
-        runner_worker = None
         orchestrator_proc_running = bool(orchestrator_proc and orchestrator_proc.is_running)
-        if not orchestrator_proc_running:
-            try:
-                from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
-
-                supervisor = get_existing_supervisord_process(quiet=True)
-                runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
-            except (OSError, RuntimeError, TimeoutError):
-                runner_worker = None
-
-        runner_worker_running = bool(runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"))
-        runner_worker_pid = runner_worker.get("pid") if runner_worker else None
-        orchestrator_running = orchestrator_proc_running or runner_worker_running
-        orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else runner_worker_pid
+        orchestrator_running = orchestrator_proc_running
+        orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else None
 
         # Get model counts by status
         crawl_status_counts = Crawl.status_counts(
             crawl_scope,
-            (Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.PAUSED),
+            (Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED),
         )
         crawls_queued = crawl_status_counts.get(Crawl.StatusChoices.QUEUED, 0)
         crawls_active = crawl_status_counts.get(Crawl.StatusChoices.STARTED, 0)
@@ -242,7 +225,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
 
         snapshot_status_counts = Snapshot.status_counts(
             snapshot_scope,
-            Snapshot.OPEN_STATES,
+            Snapshot.RUNNABLE_STATES,
         )
         snapshots_queued = snapshot_status_counts.get(Snapshot.StatusChoices.QUEUED, 0)
         snapshots_active = snapshot_status_counts.get(Snapshot.StatusChoices.STARTED, 0)
@@ -252,15 +235,24 @@ def live_progress_view(request, *, authorized_snapshot=None):
             ArchiveResult.StatusChoices.QUEUED,
             ArchiveResult.StatusChoices.STARTED,
         )
-        archiveresult_status_counts = ArchiveResult.status_counts(archiveresult_scope, result_statuses)
+        # Start from open results using the existing status index. Filtering by
+        # plugin first scans historical runs, including all completed results.
+        result_counts = list(
+            archiveresult_scope.filter(status__in=result_statuses).order_by().values("status", "plugin").annotate(count=Count("*")),
+        )
+        archiveresult_status_counts = {
+            status: sum(row["count"] for row in result_counts if row["status"] == status) for status in result_statuses
+        }
         download_scope = archiveresult_scope.filter(
             plugin__in=download_plugin_names,
             snapshot__status__in=Snapshot.RUNNABLE_STATES,
             snapshot__crawl__status__in=Crawl.RUNNABLE_STATES,
         )
-        indexing_scope = archiveresult_scope.filter(plugin__in=indexing_plugin_names)
         download_status_counts = ArchiveResult.status_counts(download_scope, result_statuses)
-        indexing_status_counts = ArchiveResult.status_counts(indexing_scope, result_statuses)
+        indexing_status_counts = {
+            status: sum(row["count"] for row in result_counts if row["status"] == status and row["plugin"] in indexing_plugin_names)
+            for status in result_statuses
+        }
         archiveresults_queued = archiveresult_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
         archiveresults_active = archiveresult_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
 
@@ -296,21 +288,27 @@ def live_progress_view(request, *, authorized_snapshot=None):
             .values(*active_crawl_fields)
             .order_by("-modified_at")[:max_active_crawls],
         )
-        paused_crawls = list(
-            crawl_scope.filter(
-                Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
-                | Q(
-                    status=Crawl.StatusChoices.PAUSED,
-                    snapshot_set__status__in=Snapshot.RUNNABLE_STATES,
-                    snapshot_set__retry_at__lte=now,
-                )
-                | Q(
-                    status=Crawl.StatusChoices.PAUSED,
-                    snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
-                ),
+        # Start with runnable children, not every historical snapshot belonging
+        # to a paused crawl. These uncorrelated subqueries run once per poll.
+        due_paused_snapshots = (
+            Snapshot.objects.filter(
+                status__in=Snapshot.RUNNABLE_STATES,
+                retry_at__lte=now,
             )
+            .order_by()
+            .values("crawl_id")
+        )
+        queued_paused_results = (
+            ArchiveResult.objects.filter(
+                status=ArchiveResult.StatusChoices.QUEUED,
+            )
+            .order_by()
+            .values("snapshot__crawl_id")
+        )
+        paused_crawls = list(
+            crawl_scope.filter(status=Crawl.StatusChoices.PAUSED)
+            .filter(Q(created_at__gte=paused_crawl_cutoff) | Q(id__in=due_paused_snapshots) | Q(id__in=queued_paused_results))
             .values(*active_crawl_fields)
-            .distinct()
             .order_by("-modified_at")[:max_active_crawls],
         )
         queued_crawls = list(
@@ -324,52 +322,29 @@ def live_progress_view(request, *, authorized_snapshot=None):
                 crawl["persona_id"] = str(crawl["persona_id"])
         persona_details_by_id: dict[str, dict[str, str]] = {}
         persona_details_by_name: dict[str, dict[str, str]] = {}
-        persona_objects_by_id = {}
-        persona_objects_by_name = {}
         persona_ids = {crawl["persona_id"] for crawl in active_crawls_list if crawl["persona_id"]}
         persona_names = {"Default"} if any(not crawl["persona_id"] for crawl in active_crawls_list) else set()
         if persona_ids or persona_names:
             from archivebox.personas.models import Persona
 
-            for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name", "config"):
+            for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name"):
                 persona_details = {
                     "name": persona.name,
                     "admin_url": f"/admin/personas/persona/{persona.pk}/change/",
                 }
                 persona_details_by_id[str(persona.id)] = persona_details
                 persona_details_by_name[persona.name] = persona_details
-                persona_objects_by_id[str(persona.id)] = persona
-                persona_objects_by_name[persona.name] = persona
         active_crawl_ids = [crawl["id"] for crawl in active_crawls_list]
-        active_crawl_objects = {}
-        if active_crawl_ids:
-            for crawl_obj in Crawl.objects.filter(id__in=active_crawl_ids).select_related("created_by", "persona"):
-                crawl_obj._runtime_config = request_config
-                active_crawl_objects[str(crawl_obj.id)] = crawl_obj
         snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in active_crawl_ids}
-        cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
-        crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
         queued_snapshot_overflow_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
         active_snapshot_scope = snapshot_scope.filter(crawl_id__in=active_crawl_ids)
         if active_crawl_ids:
-            for row in active_snapshot_scope.values("crawl_id", "status").annotate(count=Count("id")):
-                snapshot_counts_by_crawl.setdefault(str(row["crawl_id"]), {})[row["status"]] = row["count"]
-
-            for row in (
-                active_snapshot_scope.filter(status=Snapshot.StatusChoices.SEALED, downloaded_at__isnull=True)
-                .values("crawl_id")
-                .annotate(count=Count("id"))
-            ):
-                cancelled_snapshot_counts_by_crawl[str(row["crawl_id"])] = row["count"]
-
-            for row in (
-                active_snapshot_scope.filter(
-                    status=Snapshot.StatusChoices.SEALED,
-                )
-                .values("crawl_id")
-                .annotate(size=Sum("output_size"))
-            ):
-                crawl_output_sizes_by_crawl[str(row["crawl_id"])] = int(row["size"] or 0)
+            # The existing (crawl, status, modified_at) index covers these counts.
+            # Byte totals and cancellation details require table reads, so sample
+            # those separately and share cached values between viewer tabs.
+            for row in active_snapshot_scope.order_by().values("crawl_id", "status").annotate(count=Count("*")):
+                crawl_id = str(row["crawl_id"])
+                snapshot_counts_by_crawl[crawl_id][row["status"]] = row["count"]
 
         crawl_process_pids: dict[str, int] = {}
         snapshot_process_pids: dict[str, int] = {}
@@ -588,7 +563,14 @@ def live_progress_view(request, *, authorized_snapshot=None):
             completed_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.SEALED, 0)
             started_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.STARTED, 0)
             pending_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.QUEUED, 0)
-            cancelled_snapshots = cancelled_snapshot_counts_by_crawl.get(crawl_id, 0)
+            from archivebox.progressmonitor.collection import crawl_summary
+
+            totals = (
+                crawl_summary(crawl_id, snapshot_id=scoped_snapshot.pk if scoped_snapshot is not None else None)
+                if completed_snapshots
+                else {"snapshots": 0, "cancelled": 0, "bytes": 0}
+            )
+            cancelled_snapshots = totals["cancelled"] if totals is not None else None
 
             # Count URLs in the crawl (for when snapshots haven't been created yet)
             urls_count = 0
@@ -607,7 +589,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
             crawl_setup_completed = sum(1 for item in crawl_setup_plugins if item.get("status") == "succeeded")
             crawl_setup_failed = sum(1 for item in crawl_setup_plugins if item.get("status") == "failed")
             crawl_setup_pending = sum(1 for item in crawl_setup_plugins if item.get("status") == "queued")
-            crawl_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
+            crawl_screencast_url = screencast_frame_url(crawl_id)
             crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id.replace('-', '')}/change/" if crawl_screencast_url else ""
 
             # Get active snapshots for this crawl (already prefetched)
@@ -661,7 +643,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
                     snapshot_preview_url = snapshot_favicon_url
 
                 if snapshot["status"] == Snapshot.StatusChoices.STARTED:
-                    snapshot_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
+                    snapshot_screencast_url = screencast_frame_url(crawl_id)
                     snapshot_screencast_link = snapshot_view_url(snapshot) if snapshot_screencast_url else ""
 
                 def plugin_sort_key(ar):
@@ -809,18 +791,18 @@ def live_progress_view(request, *, authorized_snapshot=None):
             persona_details = persona_details_by_id.get(str(crawl["persona_id"])) if crawl["persona_id"] else None
             persona_name = persona_details["name"] if persona_details else "Default"
             persona_details = persona_details or persona_details_by_name.get(persona_name)
-            crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0)
-            avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0
-            crawl_obj = active_crawl_objects[crawl_id]
-            effective_crawl_config = get_config(crawl=crawl_obj, resolve_plugins=False)
-            max_urls = int(effective_crawl_config.CRAWL_MAX_URLS or 0)
-            crawl_max_size = int(effective_crawl_config.CRAWL_MAX_SIZE or 0)
-            crawl_timeout = int(effective_crawl_config.CRAWL_TIMEOUT or 0)
-            snapshot_max_size = int(effective_crawl_config.SNAPSHOT_MAX_SIZE or 0)
+            crawl_output_size = totals["bytes"] if totals is not None else None
+            avg_snapshot_size = int(totals["bytes"] / max(totals["snapshots"], 1)) if totals is not None else None
+            # These crawl-scope limits are frozen in Crawl.config. Rendering
+            # them must not resolve personas, probe binaries, or reread config.
+            max_urls = int(crawl["config"].get("CRAWL_MAX_URLS", request_config.CRAWL_MAX_URLS) or 0)
+            crawl_max_size = int(crawl["config"].get("CRAWL_MAX_SIZE", request_config.CRAWL_MAX_SIZE) or 0)
+            crawl_timeout = int(crawl["config"].get("CRAWL_TIMEOUT", request_config.CRAWL_TIMEOUT) or 0)
+            snapshot_max_size = int(crawl["config"].get("SNAPSHOT_MAX_SIZE", request_config.SNAPSHOT_MAX_SIZE) or 0)
 
             # Check if retry_at is in the future (would prevent worker from claiming)
             retry_at_future = crawl["retry_at"] > now if crawl["retry_at"] else False
-            is_paused = crawl_obj.is_paused
+            is_paused = crawl["status"] == Crawl.StatusChoices.PAUSED
             seconds_until_retry = (
                 0 if is_paused else int((crawl["retry_at"] - now).total_seconds()) if crawl["retry_at"] and retry_at_future else 0
             )
@@ -860,8 +842,8 @@ def live_progress_view(request, *, authorized_snapshot=None):
                     "max_snapshot_size_display": printable_filesize(snapshot_max_size) if snapshot_max_size else "unlimited",
                     "crawl_output_size": crawl_output_size,
                     "avg_snapshot_size": avg_snapshot_size,
-                    "crawl_output_size_display": printable_filesize(crawl_output_size) if crawl_output_size else "0 B",
-                    "avg_snapshot_size_display": printable_filesize(avg_snapshot_size) if avg_snapshot_size else "0 B",
+                    "crawl_output_size_display": printable_filesize(crawl_output_size or 0) if totals is not None else "…",
+                    "avg_snapshot_size_display": printable_filesize(avg_snapshot_size or 0) if totals is not None else "…",
                     "tags": crawl_tags,
                     "urls_count": urls_count,
                     "total_snapshots": total_snapshots,

archivebox/services/archive_result_service.py

@@ -210,6 +210,12 @@ def _save_archiveresult_event_to_db(
 
     with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.diff_fields"):
         update_fields = []
+        # This runner already loaded the Snapshot/Crawl for the event. Resolve
+        # retention here without extra parent reads so new results do not wait
+        # for the bounded legacy-repair sweep to traverse historical rows.
+        result.snapshot = snapshot
+        if result.set_delete_at_from_config():
+            update_fields.append("delete_at")
         for field, value in defaults.items():
             if result.__dict__[field] != value:
                 setattr(result, field, value)

archivebox/services/resource_admission.py

@@ -130,8 +130,8 @@ class ResourceAdmission:
             slots = min(slots, unpromised // observed_cost)
         return max(1, slots) if active == 0 and configured else slots
 
-    def memory_headroom(self) -> tuple[int, int, int] | None:
-        """Return workload RAM+swap usage, effective headroom, and host headroom."""
+    def memory_headroom(self) -> tuple[int, int] | None:
+        """Return workload RAM+swap usage and effective available headroom."""
         try:
             meminfo = {
                 name: int(value.split()[0]) * 1024
@@ -170,10 +170,22 @@ class ResourceAdmission:
             used_bytes = 0
             for child in processes:
                 try:
+                    if sys.platform.startswith("linux"):
+                        try:
+                            memory = child.memory_full_info()
+                            # Chrome's processes share substantial resident
+                            # memory. RSS charges those pages once per process
+                            # and can falsely serialize captures on an idle host.
+                            used_bytes += memory.pss + memory.swap
+                            continue
+                        except psutil.AccessDenied:
+                            # Restricted /proc access still gets conservative
+                            # accounting through the available RSS measurement.
+                            pass
                     used_bytes += child.memory_info().rss
                 except (psutil.Error, OSError):
                     continue
-        return used_bytes, available, host_available
+        return used_bytes, available
 
     def observe_memory_stalls(self, source: str, total: int, avg10: float, sampled_at: float) -> bool:
         previous = self._memory_stalls.get(source)

archivebox/services/runner.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import asyncio
 import contextvars
 import json
+import logging
 import os
 import signal
 import shutil
@@ -11,6 +12,7 @@ import threading
 import time
 from contextlib import nullcontext
 from datetime import timedelta
+from itertools import batched
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any, Literal
@@ -64,6 +66,7 @@ from archivebox.services.resource_admission import RESOURCE_RECHECK_SECONDS, res
 from archivebox.config.common import (
     ArchiveBoxBaseConfig,
     normalize_runtime_config,
+    _is_cookie_file_config_key,
     _plugin_enabled_config_keys,
 )
 from archivebox.misc.db import run_db_analyze_batch
@@ -223,6 +226,7 @@ class CrawlRunner:
         self.snapshot_service = SnapshotService(
             self.bus,
             crawl_id=str(crawl.id),
+            interrupted=self.execution_interrupted,
         )
         HookArchiveResultService(self.bus, emit_jsonl=False)
         ArchiveResultService(self.bus)
@@ -233,7 +237,7 @@ class CrawlRunner:
         self.snapshot_semaphore = asyncio.Semaphore(1)
         self.max_concurrent_snapshots = 1
         self._warmup_snapshot_id: str | None = None
-        self._memory_baseline: tuple[int, int] | None = None
+        self._memory_baseline: int | None = None
         self._memory_peak_cost = 0
         self._observed_snapshot_cost: int | None = None
         self._resource_deferred = False
@@ -257,6 +261,10 @@ class CrawlRunner:
         self._signal_abort_requested = False
         self._last_lease_heartbeat_at = 0.0
 
+    def execution_interrupted(self) -> bool:
+        """Owner loss preserves work; explicit user abort keeps its sealing policy."""
+        return self._signal_abort_requested and not self._user_aborted
+
     async def on_CrawlAbortEvent(self, event: CrawlAbortEvent) -> None:
         self._signal_abort_requested = True
         # An abort can arrive while the prompt transport is being established,
@@ -506,7 +514,7 @@ class CrawlRunner:
             return
         if self._observed_snapshot_cost is None and self._warmup_snapshot_id is None:
             observation = resource_admission.memory_headroom()
-            self._memory_baseline = (observation[0], observation[2]) if observation is not None else None
+            self._memory_baseline = observation[0] if observation is not None else None
             self._memory_peak_cost = 0
             self._warmup_snapshot_id = snapshot_id
         if isinstance(current_event, CrawlStartEvent):
@@ -596,15 +604,13 @@ class CrawlRunner:
             return
         observation = resource_admission.memory_headroom()
         if observation is not None:
-            used, _available, host_available = observation
-            baseline_used, baseline_host_available = self._memory_baseline
-            # Host headroom sees helpers outside ArchiveBox's cgroup. Keep a
-            # high-water cost as later URLs may use more than the first one.
-            # Never divide concurrent growth by task count: their pages vary.
+            used, _available = observation
+            # Only attribute this workload's growth to a capture. Other jobs on
+            # the host reduce available headroom, but are not a per-capture cost
+            # to multiply again for every snapshot we want to admit.
             self._memory_peak_cost = max(
                 self._memory_peak_cost,
-                used - baseline_used,
-                baseline_host_available - host_available,
+                used - self._memory_baseline,
             )
             if self._observed_snapshot_cost is not None:
                 self._observed_snapshot_cost = max(self._observed_snapshot_cost, self._memory_peak_cost)
@@ -615,6 +621,13 @@ class CrawlRunner:
         self._observe_snapshot_memory()
         if self._memory_peak_cost > 0:
             self._observed_snapshot_cost = self._memory_peak_cost
+            headroom = resource_admission.memory_headroom()
+            logging.getLogger(__name__).info(
+                "Snapshot admission: concurrency limit=%s, observed memory growth=%s bytes, available=%s bytes",
+                self.max_concurrent_snapshots,
+                self._observed_snapshot_cost,
+                headroom[1] if headroom else None,
+            )
         else:
             # No observed cost cannot justify parallel admission. Probe the
             # next real snapshot while the queue keeps moving one at a time.
@@ -716,6 +729,9 @@ class CrawlRunner:
         from archivebox.machine.models import Machine, NetworkInterface, Process
 
         self.primary_url = self.crawl.get_urls_list()[0] if self.crawl.get_urls_list() else ""
+        # Validate saved binary paths at the execution boundary, including
+        # config edited since this process first registered its Machine.
+        machine = Machine._sanitize_config(Machine.current())
         current_iface = NetworkInterface.current(refresh=not self.allow_maintenance_on_inactive_crawl)
         current_process = Process.current()
         if current_process.iface_id != current_iface.id or current_process.machine_id != current_iface.machine_id:
@@ -724,8 +740,16 @@ class CrawlRunner:
             current_process.save(update_fields=["iface", "machine", "modified_at"])
         self.persona = self.crawl.resolve_persona()
         self.base_config = get_config(crawl=self.crawl, overrides=self.config_overrides)
-        self.derived_config = dict(Machine.current().config or {})
+        # The merged user payload owns auth paths and binary selections. The
+        # install phase publishes resolved binaries; stale machine paths must
+        # not override explicit runtime settings before that phase runs.
+        self.derived_config = {
+            key: value
+            for key, value in (machine.config or {}).items()
+            if not _is_cookie_file_config_key(key) and (not key.endswith("_BINARY") or value == self.base_config.get(key))
+        }
         self.crawl_output_dir = str(self.crawl.output_dir)
+        Path(self.crawl_output_dir).mkdir(parents=True, exist_ok=True)
         if self.persona:
             self.base_config.update(
                 self.persona.prepare_runtime_for_crawl(
@@ -934,6 +958,7 @@ class CrawlRunner:
                     },
                 )
         snapshot_output_dir = str(snapshot.output_dir)
+        Path(snapshot_output_dir).mkdir(parents=True, exist_ok=True)
         tags = snapshot.tags_str()
         config = self.base_config.for_crawl_runtime(
             crawl=snapshot.crawl,
@@ -1122,18 +1147,17 @@ class CrawlRunner:
             finally:
                 cancel_watcher.cancel()
                 await asyncio.gather(cancel_watcher, return_exceptions=True)
-            completed_event = event.emit(
-                CrawlCompletedEvent(
-                    url=snapshot["url"],
-                    snapshot_id=snapshot["id"],
-                    output_dir=str(output_dir),
-                ),
-            )
-            # Same signal lifecycle as CrawlCleanupEvent above: completion is a
-            # normal bus event unless the interpreter is already unwinding from
-            # SIGINT/SIGTERM/SIGHUP, where synchronous bus delivery is no
-            # longer a dependable shutdown primitive.
-            if not self._signal_abort_requested:
+            # emit() schedules handlers immediately; guarding only the await
+            # still lets completion projectors run during takeover. An aborted
+            # execution owner must leave durable work for its replacement.
+            if not self.execution_interrupted():
+                completed_event = event.emit(
+                    CrawlCompletedEvent(
+                        url=snapshot["url"],
+                        snapshot_id=snapshot["id"],
+                        output_dir=str(output_dir),
+                    ),
+                )
                 await _run_event_now(completed_event, CrawlCompletedEvent.model_fields["event_timeout"].default)
 
         on_archivebox_CrawlStartEvent.__name__ = "on_archivebox_CrawlStartEvent__run_snapshots"
@@ -1232,6 +1256,7 @@ class CrawlRunner:
             user_config_event = MachineEvent(config=config, config_type="user")
             user_config_event.event_parent_id = crawl_start_event.event_id
             await self.bus.emit(user_config_event).now()
+            derived_config_event = None
             if derived_config:
                 derived_config_event = MachineEvent(config=derived_config, config_type="derived")
                 derived_config_event.event_parent_id = crawl_start_event.event_id
@@ -1247,6 +1272,7 @@ class CrawlRunner:
                 snapshot_cleanup_phase_timeout=snapshot_phase_timeout,
                 abort_requested=self.crawl_is_cancelled,
             )
+            projected = False
             try:
                 snapshot_event = SnapshotEvent(
                     url=snapshot["url"],
@@ -1270,6 +1296,9 @@ class CrawlRunner:
                     raise RuntimeError(f"Snapshot {snapshot_id} did not complete")
                 await completed_snapshot.wait(timeout=snapshot_phase_timeout)
                 await completed_snapshot.event_results_list()
+                projected = True
+                if self.execution_interrupted():
+                    return
                 if snapshot["status"] == "sealed":
                     await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
                     return
@@ -1291,6 +1320,16 @@ class CrawlRunner:
                 await sync_to_async(_seal_when_last_snapshot_finished, thread_sensitive=True)()
             finally:
                 snapshot_service.close()
+                # Projection and snapshot cleanup have finished. Historical
+                # capture results now live in the database; only active work
+                # needs its event ancestry available to the downloader.
+                expired = [snapshot_event, user_config_event] if projected else []
+                if projected and derived_config_event is not None:
+                    expired.append(derived_config_event)
+                if projected:
+                    expired.extend(await self.bus.filter("*", child_of=snapshot_event, past=True, future=False))
+                for event in expired:
+                    event.event_ttl = 0
 
     def seal_snapshot_due_to_limit(self, snapshot_id: str) -> None:
         from archivebox.core.models import Snapshot
@@ -1655,7 +1694,9 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
         # every URL, even though crawl isolation owns one browser for the whole
         # crawl. The crawl lifecycle lock already serializes all of its child
         # work, so let one runner drain the due siblings and discoveries.
-        if any(process.is_running for process in snapshot.process_set.filter(status="running").iterator()):
+        if any(
+            process.is_running for process in snapshot.process_set.filter(status="running").order_by("pk").paged_iterator(chunk_size=100)
+        ):
             snapshot.update_and_requeue(retry_at=timezone.now() + timedelta(seconds=lock_seconds))
             return True
         if snapshot.fs_migration_needed:
@@ -1689,7 +1730,7 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
     if not snapshot.claim_processing_lock(lock_seconds=lock_seconds):
         return False
     snapshot.refresh_from_db()
-    if any(process.is_running for process in snapshot.process_set.filter(status="running").iterator()):
+    if any(process.is_running for process in snapshot.process_set.filter(status="running").order_by("pk").paged_iterator(chunk_size=100)):
         # The Snapshot lease may have expired while an abx-dl hook process is
         # still alive. Preserve the snapshot-level ownership boundary and do
         # not launch a second sequence; ArchiveResult status is irrelevant.
@@ -1863,7 +1904,24 @@ def run_install(*, plugin_names: list[str] | None = None) -> None:
 
 def _first_due_id(queryset, *, newest_first: bool = False):
     ordering = ("-retry_at", "-created_at") if newest_first else ("retry_at", "created_at")
-    return queryset.order_by(*ordering).values_list("id", flat=True).first()
+    # Queue membership comes from retry_at, not status: almost the entire
+    # archive can be sealed while only a handful of rows need maintenance.
+    # Combining both predicates lets SQLite choose the broad status index and
+    # scan/sort archival history on every idle pass (567ms on Cabbage vs 3ms
+    # through retry_at). First materialize bounded due IDs using the existing
+    # queue index, then apply status/parent filters to those IDs. This also
+    # bounds PostgreSQL reads and keeps every cursor closed before a CAS claim.
+    due_ids = (
+        queryset.model.objects.filter(retry_at__lte=timezone.now())
+        .order_by(*ordering)
+        .values_list("id", flat=True)
+        .paged_iterator(chunk_size=100)
+    )
+    for candidate_ids in batched(due_ids, 100):
+        match = queryset.filter(id__in=candidate_ids).order_by(*ordering).values_list("id", flat=True).first()
+        if match is not None:
+            return match
+    return None
 
 
 def _run_due_crawl_status(
@@ -1934,12 +1992,8 @@ def _run_due_snapshot_id(snapshot_id, *, lock_seconds: int, interactive_interrup
 def _run_due_binary() -> bool:
     from archivebox.machine.models import Binary
 
-    due_binary_id = (
-        Binary.objects.filter(retry_at__lte=timezone.now())
-        .exclude(status=Binary.StatusChoices.INSTALLED)
-        .order_by("retry_at", "created_at")
-        .values_list("id", flat=True)
-        .first()
+    due_binary_id = _first_due_id(
+        Binary.objects.filter(retry_at__lte=timezone.now()).exclude(status=Binary.StatusChoices.INSTALLED),
     )
     if due_binary_id is None:
         return False
@@ -1966,14 +2020,36 @@ def run_pending_crawls(
     last_recovery_at = 0.0
     last_retention_at = 0.0
     last_retention_repair_at = 0.0
+    retention_backfill_cursors: dict = {}
+    deletion_thread: threading.Thread | None = None
     last_analyze_at = 0.0
     analyze_queue: list[str] | None = None
     analyze_sweep_started_at = 0.0
     orchestrator_started_at = time.monotonic()
+
+    def delete_in_background() -> None:
+        from django.db import connections
+
+        try:
+            Snapshot.delete_requested(batch_size=100)
+        finally:
+            connections.close_all()
+
     while True:
         raise_if_shutdown_requested()
         now_monotonic = time.monotonic()
         if crawl_id is None and now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
+            if daemon:
+                # At most one batch can be doing filesystem I/O. A stuck mount
+                # must not block capture scheduling or runner shutdown; rows
+                # remain pending until cleanup succeeds, including on restart.
+                if (deletion_thread is None or not deletion_thread.is_alive()) and Snapshot.objects.filter(
+                    status=Snapshot.DELETING_STATE,
+                ).exists():
+                    deletion_thread = threading.Thread(target=delete_in_background, name="snapshot-delete", daemon=True)
+                    deletion_thread.start()
+            else:
+                Snapshot.delete_requested(batch_size=100)
             for model in (ArchiveResult, Snapshot, Crawl, Process):
                 # Keep the tight scheduler loop anchored on indexed delete_at
                 # columns only. Backfilling missing delete_at values has to read
@@ -2141,7 +2217,7 @@ def run_pending_crawls(
                 # delete_at in the plugin-result hot path. Running it here keeps
                 # DELETE_AFTER resolution fresh without making every hook event
                 # load parent Snapshot/Crawl config.
-                model.delete_expired(batch_size=100, backfill_missing=True)
+                model.delete_expired(batch_size=100, backfill_missing=True, backfill_cursors=retention_backfill_cursors)
             last_retention_repair_at = now_monotonic
 
         if daemon:
@@ -2157,12 +2233,10 @@ def run_pending_crawls(
             # snapshot detail page out to ~500ms. Refresh stats at most once per
             # 24hr while the queue is idle, and only after the orchestrator has
             # been alive for at least an hour so short server boots / one-off work
-            # never pay the cost. The sweep is batched one table per idle tick;
-            # individual table ANALYZE statements abort after 2min (progress
-            # handler) and the whole sweep is hard-capped at 5min so a
-            # pathological table cannot wedge maintenance forever. Any failure
-            # inside the maintenance hook is swallowed — orchestrator must never
-            # be taken down by stats refresh.
+            # never pay the cost. Each idle tick samples a bounded number of
+            # index entries for one table; never scan the full history while
+            # holding ANALYZE's write transaction. Optional stats refresh must
+            # not prevent queued captures.
             try:
                 if (
                     analyze_queue is None

archivebox/services/snapshot_service.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+from collections.abc import Callable
 from pathlib import Path
 
 from asgiref.sync import sync_to_async
@@ -73,10 +74,11 @@ def finalize_completed_snapshot(
             modified_at=timezone.now(),
         )
 
-    # SnapshotCompletedEvent is abx-dl's authoritative signal that the complete
-    # snapshot hook sequence (including cleanup) finished. ArchiveResult rows
-    # are projections of that work, never prerequisites used to decide whether
-    # the Snapshot may seal.
+    # SnapshotCompletedEvent finishes an attempt, including abort cleanup. An
+    # explicit user abort deliberately seals that point-in-time capture; the
+    # runner's interrupted-owner guard below keeps takeover work resumable.
+    # ArchiveResult rows are projections of that work, never prerequisites used
+    # to decide whether the Snapshot may seal.
     if not was_sealed and snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.retry_at == owned_retry_at:
         snapshot.seal()
         snapshot.refresh_from_db()
@@ -120,8 +122,9 @@ class SnapshotService(BaseService):
     LISTENS_TO = [SnapshotEvent, SnapshotCompletedEvent]
     EMITS = []
 
-    def __init__(self, bus, *, crawl_id: str):
+    def __init__(self, bus, *, crawl_id: str, interrupted: Callable[[], bool] | None = None):
         self.crawl_id = crawl_id
+        self.interrupted = interrupted
         self._run_ownership: dict[str, tuple[str, object, bool, list[str]]] = {}
         self._ownership_lock = asyncio.Lock()
         super().__init__(bus)
@@ -143,7 +146,7 @@ class SnapshotService(BaseService):
             elif snapshot.status not in (Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED):
                 return
             if snapshot.status == Snapshot.StatusChoices.STARTED:
-                await sync_to_async(snapshot.ensure_crawl_symlink, thread_sensitive=True)()
+                await sync_to_async(snapshot.reconcile_filesystem_links, thread_sensitive=True)()
             retry_plugins = [str(name).strip() for name in (snapshot.config or {}).get("RETRY_PLUGINS", []) if str(name).strip()]
             async with self._ownership_lock:
                 self._run_ownership[str(event.snapshot_id)] = (str(event.event_id), snapshot.retry_at, was_sealed, retry_plugins)
@@ -175,6 +178,15 @@ class SnapshotService(BaseService):
             if ownership is None or ownership[0] != str(event.event_parent_id):
                 return
             self._run_ownership.pop(snapshot_id, None)
+        # abx-dl also emits completion after abort cleanup, even before the
+        # first hook. Explicit user aborts seal a point-in-time capture so later
+        # work cannot silently mix observations taken days apart; reopening it
+        # is a separate user action. Takeover only replaces its execution owner.
+        # Consult the runner's synchronous signal state: an exiting owner cannot
+        # rely on another bus event arriving before this completion projector runs.
+        # Leave its durable lease for the next elected runner's startup recovery.
+        if self.interrupted is not None and self.interrupted():
+            return
         _, owned_retry_at, was_sealed, retry_plugins = ownership
         await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(
             event.snapshot_id,

archivebox/templates/admin/base.html

@@ -1527,6 +1527,9 @@
                 display: inline-flex;
                 flex-wrap: wrap;
                 align-items: center;
+                min-width: 0;
+                max-width: 100%;
+                box-sizing: border-box;
                 gap: 4px;
                 padding: 2px 4px;
                 background: transparent;
@@ -1540,15 +1543,27 @@
                 flex-wrap: wrap;
                 gap: 4px;
                 align-items: center;
+                min-width: 0;
+                max-width: 100%;
             }
 
             .tag-editor-inline .tag-pill {
+                min-width: 0;
+                max-width: min(160px, 100%);
+                box-sizing: border-box;
                 padding: 2px 6px 2px 8px;
                 font-size: 11px;
                 border-radius: 12px;
             }
 
+            .tag-editor-inline .tag-link {
+                min-width: 0;
+                overflow: hidden;
+                text-overflow: ellipsis;
+            }
+
             .tag-editor-inline .tag-remove-btn {
+                flex-shrink: 0;
                 width: 14px;
                 height: 14px;
                 font-size: 12px;

archivebox/templates/admin/snapshots_grid.html

@@ -79,6 +79,13 @@
   }
   .cards .card .card-tags span {
       display: inline-block;
+      max-width: min(160px, 100%);
+      box-sizing: border-box;
+      overflow: hidden;
+      text-overflow: ellipsis;
+      white-space: nowrap;
+      vertical-align: middle;
+      pointer-events: auto;
       padding: 2px 5px;
       border-radius: 5px;
       opacity: 0.95;
@@ -283,16 +290,16 @@
           <div class="card-media">
             <a href="{% snapshot_detail_url obj %}" class="card-thumbnail">
               {% snapshot_thumbnail obj admin=True width="100%" height="245px" %}
-            </a>
               {% if obj.tags_str %}
                 <div class="card-tags">
                   {% for tag in obj.tags_str|split:',' %}
-                    <span>
+                    <span title="{{tag}}">
                       {{tag}}
                     </span>
                   {% endfor %}
                 </div>
               {% endif %}
+            </a>
           </div>
           <div class="card-outputs">
             {% if obj.status == 'started' %}

archivebox/templates/core/public_index.html

@@ -198,21 +198,29 @@
     }
 
     .snapshot-preview-cell {
+        position: relative;
         width: 110px;
         min-width: 116px;
         vertical-align: middle;
     }
 
     .snapshot-preview-cell a {
+        position: absolute;
+        top: 10px;
+        bottom: 10px;
         display: inline-flex;
         align-items: center;
         justify-content: center;
         width: 102px;
-        height: 102px;
         line-height: 0;
         vertical-align: middle;
     }
 
+    .snapshot-preview-cell .snapshot-thumbnail {
+        width: 100% !important;
+        height: 100% !important;
+    }
+
     #content .public-snapshot-list .snapshot-preview-cell .snapshot-preview,
     #content .public-snapshot-list .snapshot-preview-cell img.snapshot-preview.screenshot {
         box-sizing: border-box;
@@ -330,9 +338,10 @@
     }
 
     .snapshot-tag {
-        display: inline-flex;
-        align-items: center;
-        max-width: 160px;
+        display: block;
+        min-width: 0;
+        max-width: min(160px, 100%);
+        box-sizing: border-box;
         padding: 2px 7px;
         border: 1px solid #bfdbfe;
         border-radius: 999px;
@@ -585,7 +594,8 @@
 
         .snapshot-preview-cell a {
             width: 82px;
-            height: 82px;
+            top: 7px;
+            bottom: 7px;
         }
 
         #content .public-snapshot-list .snapshot-preview-cell .snapshot-preview,
@@ -644,12 +654,19 @@
                 </span>
                 <input type="submit" value="Search">
             </form>
+            {% comment %}
+                Missing/stale totals are intentional on large collections: show rows
+                immediately and let a later normal reload pick up the cached count.
+                Do not fill this in with an extra count request or paginator.count:
+                the unfiltered paginator only knows the rows needed for navigation.
+                Use the optional public_* totals here and in the footer below.
+            {% endcomment %}
             <div class="public-snapshot-count">
                 {% if public_search_stream_pending %}
                     <span class="archivebox-search-stream-status" role="status" aria-live="polite">Searching...</span>
                 {% else %}
-                    {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }}
-                    <span>(page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }})</span>
+                    {{ page_obj.start_index }}-{{ page_obj.end_index }}{% if public_snapshot_count != None %} of {{ public_snapshot_count }}{% endif %}
+                    <span>(page {{ page_obj.number }}{% if public_page_count != None %} of {{ public_page_count }}{% endif %})</span>
                 {% endif %}
             </div>
         </div>
@@ -659,7 +676,7 @@
                     <tr>
                         <th class="snapshot-time">Saved</th>
                         <th class="snapshot-preview-cell">Preview</th>
-                        <th>Snapshot ({{ page_obj.paginator.count }})</th>
+                        <th>Snapshot{% if public_snapshot_count != None %} ({{ public_snapshot_count }}){% endif %}</th>
                         <th class="snapshot-tags-cell">Tags</th>
                         <th class="snapshot-status-cell">Status</th>
                         <th class="snapshot-files-cell">Files</th>
@@ -687,7 +704,7 @@
             </table>
         </div>
         <div class="public-pagination">
-            Showing {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
+            Showing {{ page_obj.start_index }}-{{ page_obj.end_index }}{% if public_snapshot_count != None %} of {{ public_snapshot_count }} total{% endif %}
             <br>
             <span class="step-links">
                 {% if page_obj.has_previous %}
@@ -696,14 +713,14 @@
                 {% endif %}
 
                 <span class="current">
-                    Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}
+                    Page {{ page_obj.number }}{% if public_page_count != None %} of {{ public_page_count }}{% endif %}
                 </span>
 
                 {% if page_obj.has_next %}
                     <a href="?{% url_replace page=page_obj.next_page_number %}">next</a>
                 {% endif %}
-                {% if page_obj.has_next %}
-                    <a href="?{% url_replace page=page_obj.paginator.num_pages %}">last &raquo;</a>
+                {% if page_obj.has_next and public_page_count != None %}
+                    <a href="?{% url_replace page=public_page_count %}">last &raquo;</a>
                 {% endif %}
             </span>
         </div>

archivebox/templates/core/setup_wizard.html

@@ -35,7 +35,7 @@
       <div class="abx-question-options" id="archivebox-setup-dns-options">
         <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="localhost"> <span><b>⭐ Localhost</b><small>No setup needed. <code id="archivebox-setup-localhost-dns-example">*.archivebox.localhost</code> works directly in modern browsers with all features.</small></span></label>
         <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="single"> <span><b>⚠️ Single-domain DNS</b><small>Register a DNS record like <code>A archivebox.example.com</code> → <code>your server's IP</code>, or add an equivalent <code>/etc/hosts</code> entry pointing to your server. Some archived content that depends on JavaScript may not display properly in this mode. JavaScript still runs during capture, but viewing HTML captured by <code>wget</code>, <code>dom</code>, <code>responses</code>, <code>staticfile</code>, and similar plugins will not replay JavaScript unless wildcard DNS is used.</small></span></label>
-        <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="wildcard"> <span><b>⭐ Wildcard DNS</b><small>Point both the bare <code>BASE_URL</code> hostname and <code>*.BASE_URL</code> to this server using A/AAAA records or CNAME records. Ideal: safest isolation with all replay features. <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes" target="_blank" rel="noopener noreferrer">Setup instructions</a>.</small></span></label>
+        <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="wildcard"> <span><b>⭐ Wildcard DNS</b><small>Point both the bare <code>BASE_URL</code> hostname and <code>*.BASE_URL</code> to this server using A/AAAA records or CNAME records. Ideal: safest isolation with all replay features. <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode" target="_blank" rel="noopener noreferrer">Setup instructions</a>.</small></span></label>
       </div>
       <div class="abx-question-status" id="archivebox-setup-dns-status">Choose a DNS mode to begin testing.</div>
     </fieldset>

archivebox/templates/core/snapshot.html

@@ -1671,6 +1671,13 @@
                     return false
                 }
 
+                // A new snapshot has only the synthetic "Browse all files" card.
+                // Selecting it would replace the live monitor with an empty index.
+                // Keep progress/empty state until the user explicitly opens a file.
+                if (document.getElementById('main-frame-wrapper')?.dataset.hasOutputs !== '1') {
+                    return false
+                }
+
                 const selectedCard = document.querySelector('.thumb-card.selected-card') || document.querySelector('.thumb-card')
                 const selectedLink = selectedCard && (selectedCard.querySelector('a[target=preview]') || selectedCard.querySelector('a'))
                 if (selectedCard && selectedLink) {

archivebox/templates/core/system_warnings_banner.html

@@ -27,7 +27,7 @@
       Only <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ free_gb }} GiB</code>
       free on <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">DATA_DIR</code>
       — new archive jobs will fail until space is reclaimed.
-      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Disk-Usage" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
+      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#purging-entries" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
     {% elif mode == "high_memory" %}
       Virtual memory at <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ mem_pct }}%</code>
       — the host is close to OOM. Consider stopping crawls or scaling up.
@@ -35,7 +35,7 @@
       15-min loadavg <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ load_15 }}</code>
     {% elif mode == "unsafe" %}
       ArchiveBox single-domain mode — archived pages share an origin with this site
-      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
+      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
     {% elif mode == "base_url_mismatch" %}
       Editing disabled from unrecognized URL. Visit
       <a href="{{ configured_base_url }}" style="color:#fff;text-decoration:underline;text-transform:none;">{{ configured_base_url }}</a>

archivebox/templates/static/admin-inline-tags.js

@@ -85,6 +85,7 @@
             link.href = '/admin/core/snapshot/?tags__id__exact=' + td.id;
             link.className = 'tag-link';
             link.textContent = td.name;
+            link.title = td.name;
             pill.appendChild(link);
 
             var removeBtn = document.createElement('button');

archivebox/templates/static/admin.css

@@ -1155,7 +1155,19 @@ body.change-list:not(.model-snapshot) #changelist #toolbar,
     opacity: 0.75;
 }
 
+#result_list tbody td.field-preview_icon {
+    position: relative;
+}
+
+#result_list tbody td.field-preview_icon > a {
+    position: absolute;
+    inset: 4px 2px;
+    display: block;
+}
+
 #result_list tbody td.field-preview_icon .snapshot-thumbnail {
+    width: 100% !important;
+    height: 100% !important;
     transition: box-shadow 140ms ease, transform 140ms ease;
 }
 
@@ -2226,7 +2238,7 @@ tbody .output-link:hover {opacity: 1;}
     .model-snapshot.change-list #result_list tbody th.field-created_at_display > a,
     .model-snapshot.change-list #result_list tbody td.field-created_at_display > a {
         display: inline-flex;
-        max-width: none;
+        max-width: 100%;
         white-space: nowrap !important;
     }
 
@@ -2234,8 +2246,10 @@ tbody .output-link:hover {opacity: 1;}
     .model-snapshot.change-list #result_list tbody td.field-created_at_display .snapshot-created-at {
         display: inline-flex;
         flex-direction: row !important;
+        flex-wrap: wrap;
         justify-content: center;
         gap: 5px;
+        row-gap: 0;
         white-space: nowrap !important;
     }
 
@@ -2250,13 +2264,10 @@ tbody .output-link:hover {opacity: 1;}
         grid-area: preview;
         align-self: stretch;
         place-items: center;
-        min-height: 110px;
         overflow: hidden;
     }
 
     .model-snapshot.change-list #result_list tbody td.field-preview_icon .snapshot-thumbnail {
-        width: 80px !important;
-        height: 110px !important;
         max-width: 100%;
         box-sizing: border-box;
     }
@@ -2278,7 +2289,7 @@ tbody .output-link:hover {opacity: 1;}
         padding-left: 0;
     }
 
-    .model-snapshot.change-list #result_list tbody td.field-title_str a {
+    .model-snapshot.change-list #result_list tbody td.field-title_str > a {
         display: block;
         max-width: 100%;
         overflow-wrap: anywhere;
@@ -2514,26 +2525,12 @@ tbody .output-link:hover {opacity: 1;}
     }
 
     .model-snapshot.change-list #result_list tbody tr {
-        grid-template-columns: 24px 84px minmax(0, 1fr);
         gap: 2px 4px;
-        grid-template-areas:
-            "check preview date"
-            "check preview title"
-            "check preview tags"
-            "check preview status"
-            "check preview files"
-            "check preview size";
     }
 
     .model-snapshot.change-list #result_list tbody td.field-files .files-icons {
         gap: 0;
     }
-
-    .model-snapshot.change-list #result_list tbody td.field-status_with_progress,
-    .model-snapshot.change-list #result_list tbody td.field-size_with_stats {
-        justify-self: start;
-        text-align: left;
-    }
 }
 
 body.embedded-change-list #content {

archivebox/templates/static/directory_index.html

@@ -1,357 +1,177 @@
 {% load i18n %}
+{% get_current_language as language_code %}
 <!DOCTYPE html>
-<html lang="en">
+<html lang="{{ language_code }}">
   <head>
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-    <meta http-equiv="Content-Language" content="en-us">
+    <meta http-equiv="Content-Language" content="{{ language_code }}">
     <meta name="robots" content="NONE,NOARCHIVE">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <title>{% blocktranslate %}Index of {{ directory }}{% endblocktranslate %}</title>
     <style>
-      :root {
-        color-scheme: light;
-        --bg: #f3f5f8;
-        --panel: #ffffff;
-        --panel-border: #d9e0e8;
-        --text: #16202a;
-        --muted: #5e6b78;
-        --accent: #0f5cc0;
-        --accent-soft: #e7f0ff;
-        --folder-bg: #f6f9ff;
-        --folder-border: #d6e5ff;
-        --shadow: 0 10px 30px rgba(20, 32, 52, 0.08);
-      }
-
-      * {
-        box-sizing: border-box;
-      }
-
-      html,
-      body {
-        margin: 0;
-        padding: 0;
-        background: linear-gradient(180deg, #eef3f9 0%, var(--bg) 100%);
-        color: var(--text);
-        font-family:
-          ui-sans-serif,
-          system-ui,
-          -apple-system,
-          BlinkMacSystemFont,
-          "Segoe UI",
-          sans-serif;
-      }
-
-      body {
-        min-height: 100vh;
-        padding: 24px;
-      }
-
-      .directory-shell {
-        width: min(980px, 100%);
-        margin: 0 auto;
-      }
-
-      .directory-toolbar {
-        position: sticky;
-        top: 0;
-        z-index: 10;
-        display: flex;
-        flex-wrap: wrap;
-        gap: 10px;
-        margin-bottom: 14px;
-        padding: 12px;
-        border: 1px solid var(--panel-border);
-        border-radius: 16px;
-        background: rgba(255, 255, 255, 0.9);
-        box-shadow: var(--shadow);
-        backdrop-filter: blur(10px);
-      }
-
-      .toolbar-link {
-        display: inline-flex;
-        align-items: center;
-        gap: 8px;
-        min-height: 40px;
-        padding: 0 14px;
-        border: 1px solid var(--panel-border);
-        border-radius: 999px;
-        background: #fff;
-        color: var(--text);
-        font-size: 14px;
-        font-weight: 600;
-        text-decoration: none;
-        transition:
-          background-color 0.15s ease,
-          border-color 0.15s ease,
-          transform 0.15s ease;
-      }
-
-      .toolbar-link:hover {
-        background: #f8fbff;
-        border-color: #c5d2e0;
-        transform: translateY(-1px);
-      }
-
-      .toolbar-link.primary {
-        background: var(--accent-soft);
-        border-color: var(--folder-border);
-        color: var(--accent);
-      }
-
-      .archivebox-zip-spinner {
-        display: none;
-        width: 14px;
-        height: 14px;
-        border: 2px solid #2563eb;
-        border-right-color: transparent;
-        border-radius: 999px;
-        animation: archivebox-zip-spin 0.75s linear infinite;
-        flex: 0 0 auto;
-        box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.45);
-      }
-
-      .archivebox-zip-button.is-loading {
-        pointer-events: none;
-        opacity: 0.96;
-      }
-
-      .archivebox-zip-button.is-loading .archivebox-zip-spinner {
-        display: inline-block;
-      }
-
-      .archivebox-zip-button.is-loading .archivebox-zip-label {
-        opacity: 0.72;
-      }
-
-      @keyframes archivebox-zip-spin {
-        to {
-          transform: rotate(360deg);
-        }
-      }
-
-      .directory-card {
-        background: var(--panel);
-        border: 1px solid var(--panel-border);
-        border-radius: 18px;
-        box-shadow: var(--shadow);
-        overflow: hidden;
-      }
-
-      .directory-header {
-        padding: 22px 24px 18px;
-        border-bottom: 1px solid var(--panel-border);
-        background:
-          radial-gradient(circle at top right, rgba(15, 92, 192, 0.08), transparent 28%),
-          linear-gradient(180deg, #ffffff, #f8fbff);
-      }
-
-      .directory-kicker {
-        margin: 0 0 8px;
-        color: var(--muted);
-        font-size: 12px;
-        font-weight: 700;
-        letter-spacing: 0.08em;
-        text-transform: uppercase;
-      }
-
-      .directory-title {
-        margin: 0;
-        font-size: clamp(22px, 4vw, 34px);
-        line-height: 1.1;
-        word-break: break-word;
-      }
-
-      .directory-meta {
-        margin-top: 12px;
-        color: var(--muted);
-        font-size: 14px;
-      }
-
-      .directory-path {
-        display: inline-block;
-        margin-top: 12px;
-        padding: 8px 12px;
-        border-radius: 999px;
-        background: #f4f7fb;
-        border: 1px solid var(--panel-border);
-        color: #304152;
-        font-family:
-          ui-monospace,
-          "SFMono-Regular",
-          Menlo,
-          Consolas,
-          monospace;
-        font-size: 13px;
-      }
-
-      .directory-list {
-        list-style: none;
-        margin: 0;
-        padding: 12px;
-      }
-
-      .directory-entry + .directory-entry {
-        margin-top: 6px;
-      }
-
-      .directory-link {
-        display: flex;
-        align-items: center;
-        gap: 12px;
-        width: 100%;
-        padding: 12px 14px;
-        border: 1px solid transparent;
-        border-radius: 12px;
-        color: inherit;
-        text-decoration: none;
-        transition:
-          background-color 0.15s ease,
-          border-color 0.15s ease,
-          transform 0.15s ease,
-          box-shadow 0.15s ease;
-      }
-
-      .directory-link:hover {
-        background: #f8fbff;
-        border-color: var(--panel-border);
-        transform: translateY(-1px);
-      }
-
-      .directory-link:focus-visible {
-        outline: 3px solid rgba(15, 92, 192, 0.2);
-        outline-offset: 1px;
-      }
-
-      .directory-entry.folder .directory-link {
-        background: var(--folder-bg);
-        border-color: var(--folder-border);
-      }
-
-      .entry-icon {
-        flex: 0 0 auto;
-        width: 34px;
-        height: 34px;
-        display: inline-flex;
-        align-items: center;
-        justify-content: center;
-        border-radius: 10px;
-        background: #edf2f8;
-        font-size: 18px;
-      }
-
-      .directory-entry.folder .entry-icon {
-        background: var(--accent-soft);
-      }
-
-      .entry-label {
-        min-width: 0;
-        display: flex;
-        flex-direction: column;
-        gap: 3px;
-      }
-
-      .entry-name {
-        font-size: 15px;
-        font-weight: 600;
-        line-height: 1.35;
-        word-break: break-word;
-      }
-
-      .entry-kind {
-        color: var(--muted);
-        font-size: 12px;
-        line-height: 1.2;
-      }
-
-      .directory-entry.parent .entry-kind {
-        color: var(--accent);
-      }
-
-      .empty-state {
-        padding: 28px 24px 32px;
-        text-align: center;
-        color: var(--muted);
-      }
-
-      @media (max-width: 640px) {
-        body {
-          padding: 14px;
-        }
-
-        .directory-header {
-          padding: 18px 18px 16px;
-        }
-
-        .directory-list {
-          padding: 10px;
-        }
-
-        .directory-link {
-          padding: 11px 12px;
-        }
-      }
+      :root { color-scheme: light; --line: #dfe4eb; --text: #253247; --muted: #667085; --accent: #245dc1; }
+      * { box-sizing: border-box; }
+      html, body { margin: 0; padding: 0; min-height: 100%; background: white; color: var(--text); font: 13px/1.4 system-ui, sans-serif; }
+      main { width: 100%; }
+      a { color: inherit; text-decoration: none; }
+      .directory-toolbar { display: flex; flex-wrap: wrap; gap: 6px; padding: 8px 12px; border-bottom: 1px solid var(--line); }
+      .toolbar-link, .entry-download { display: inline-flex; align-items: center; justify-content: center; gap: 5px; height: 30px; padding: 4px 9px; border: 1px solid var(--line); border-radius: 4px; background: white; }
+      .toolbar-link:hover, .entry-download:hover { background: #f0f5ff; }
+      .directory-header { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; padding: 8px 12px; border-bottom: 1px solid var(--line); background: #f8fafc; }
+      .directory-title { margin: 0; font-size: 13px; font-weight: 600; overflow-wrap: anywhere; }
+      .directory-meta { margin-left: auto; color: var(--muted); white-space: nowrap; }
+      #filter-files { min-width: 0; width: 180px; padding: 5px 8px; border: 1px solid var(--line); border-radius: 4px; font: inherit; }
+      .directory-list { width: 100%; border-collapse: collapse; table-layout: fixed; }
+      th { height: 30px; padding: 0; text-align: left; background: #f8fafc; border-bottom: 1px solid var(--line); }
+      th:first-child button { padding-left: 72px; }
+      th button { width: 100%; height: 30px; padding: 0 8px; border: 0; background: none; color: var(--muted); text-align: left; font: inherit; cursor: pointer; }
+      th[aria-sort="ascending"] button::after { content: " ↑"; }
+      th[aria-sort="descending"] button::after { content: " ↓"; }
+      .size-column { width: 90px; }
+      .type-column { width: 220px; }
+      .download-column { width: 50px; }
+      td { padding: 0; }
+      .entry-size, .entry-type { padding: 0 8px; color: var(--muted); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+      .entry-size { font-variant-numeric: tabular-nums; }
+      .directory-entry { height: 50px; border-bottom: 1px solid var(--line); }
+      .directory-entry[hidden] { display: none; }
+      .directory-entry:hover { background: #f8fafc; }
+      .directory-link { display: grid; grid-template-columns: 48px minmax(0, 1fr); align-items: center; gap: 12px; min-width: 0; height: 49px; padding: 6px 12px; }
+      a:focus-visible { outline: 2px solid var(--accent); outline-offset: -2px; }
+      .entry-icon, .entry-preview { width: 48px; height: 36px; margin: 0; overflow: hidden; border-radius: 3px; }
+      .entry-icon { display: grid; place-items: center; font-size: 20px; }
+      .entry-preview { border: 1px solid var(--line); background: white; }
+      img.entry-preview { object-fit: contain; }
+      pre.entry-preview { padding: 3px; white-space: pre-wrap; overflow-wrap: anywhere; font: 7px/1.2 ui-monospace, monospace; color: var(--muted); }
+      .entry-label { min-width: 0; display: flex; flex-direction: column; gap: 1px; }
+      .entry-name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-weight: 500; }
+      .entry-kind { color: var(--muted); font-size: 11px; }
+      .entry-download { flex-shrink: 0; color: var(--accent); }
+      .empty-state { padding: 24px; text-align: center; color: var(--muted); }
+      .archivebox-zip-spinner { display: none; width: 12px; height: 12px; border: 2px solid var(--accent); border-right-color: transparent; border-radius: 50%; animation: archivebox-zip-spin .75s linear infinite; }
+      .archivebox-zip-button.is-loading { pointer-events: none; opacity: .7; }
+      .archivebox-zip-button.is-loading .archivebox-zip-spinner { display: inline-block; }
+      @keyframes archivebox-zip-spin { to { transform: rotate(360deg); } }
+      @media (max-width: 640px) { .size-column { width: 65px; } .type-column { width: 100px; } .download-column { width: 40px; } #filter-files { width: 130px; } }
     </style>
   </head>
   <body>
-    <main class="directory-shell">
-      <nav class="directory-toolbar" aria-label="Directory navigation">
+    <main>
+      <nav class="directory-toolbar" aria-label="{% translate "Directory navigation" %}">
         {% if directory != "/" %}
-          <a class="toolbar-link" href="../{{ directory_query }}">↩ Up One Level</a>
+          <a class="toolbar-link" href="../{{ directory_query }}">↩ {% translate "Up One Level" %}</a>
         {% endif %}
-        <a class="toolbar-link primary" href="{{ snapshot_page_url }}">⌂ Snapshot Page</a>
+        <a class="toolbar-link" href="{{ snapshot_page_url }}">⌂ {% translate "Snapshot Page" %}</a>
         <a
           class="toolbar-link archivebox-zip-button"
           href="{{ zip_url }}"
-          data-loading-label="Preparing..."
+          data-loading-label="{% translate "Preparing..." %}"
           onclick="return window.archiveboxHandleZipClick(this, event);"
         >
           <span class="archivebox-zip-spinner" aria-hidden="true"></span>
-          <span class="archivebox-zip-label">⬇ Download Zip</span>
+          <span class="archivebox-zip-label">⬇ {% translate "Download Zip" %}</span>
         </a>
       </nav>
 
-      <section class="directory-card">
+      <section>
         <header class="directory-header">
-          <p class="directory-kicker">ArchiveBox File Index</p>
-          <h1 class="directory-title">{% blocktranslate %}Index of {{ directory }}{% endblocktranslate %}</h1>
-          <div class="directory-meta">
-            {{ file_list|length }} item{{ file_list|length|pluralize }}
+          <h1 class="directory-title">{{ directory }}</h1>
+          <div class="directory-meta" id="file-count" aria-live="polite">
+            {% translate "Items" %}: <span id="visible-file-count">{{ file_list|length }}</span>
           </div>
-          <div class="directory-path">{{ directory }}</div>
+          <input id="filter-files" type="search" aria-label="{% translate "Filter files" %}" placeholder="{% translate "Filter files" %}">
         </header>
 
         {% if file_list or directory != "/" %}
-          <ul class="directory-list">
+          <table class="directory-list">
+            <colgroup><col><col class="size-column"><col class="type-column"><col class="download-column"></colgroup>
+            <thead><tr>
+              <th scope="col" aria-sort="ascending"><button type="button" data-sort="name" aria-label="{% translate "Name" %}">{% translate "Name" %}</button></th>
+              <th scope="col" aria-sort="none"><button type="button" data-sort="size" aria-label="{% translate "Size" %}">{% translate "Size" %}</button></th>
+              <th scope="col" aria-sort="none"><button type="button" data-sort="type" aria-label="{% translate "Type" %}">{% translate "Type" %}</button></th>
+              <th scope="col" aria-label="{% translate "Download" %}"></th>
+            </tr></thead>
+            <tbody>
             {% if directory != "/" %}
-              <li class="directory-entry parent">
+              <tr class="directory-entry parent"><td>
                 <a class="directory-link" href="../{{ directory_query }}">
                   <span class="entry-icon" aria-hidden="true">↩</span>
                   <span class="entry-label">
                     <span class="entry-name">../</span>
-                    <span class="entry-kind">Parent directory</span>
                   </span>
                 </a>
-              </li>
+              </td><td class="entry-size">—</td><td class="entry-type">{% translate "Directory" %}</td><td></td></tr>
             {% endif %}
 
-            {% for f in file_list %}
-              <li class="directory-entry{% if f|slice:'-1:' == '/' %} folder{% endif %}">
-                <a class="directory-link" href="{{ f|urlencode }}{% if f|slice:'-1:' == '/' %}{{ directory_query }}{% endif %}">
-                  <span class="entry-icon" aria-hidden="true">{% if f|slice:'-1:' == '/' %}📁{% else %}📄{% endif %}</span>
+            {% for entry in entries %}
+              {% with f=entry.name %}
+              <tr class="directory-entry{% if entry.is_dir %} folder{% endif %}" data-name="{{ f }}" data-size="{{ entry.size_bytes }}" data-type="{{ entry.mimetype }}"><td>
+                <a class="directory-link" href="{{ f|urlencode }}{% if f|slice:'-1:' == '/' %}{{ directory_query }}{% elif f|lower|slice:'-4:' == '.zip' %}?preview=1{% endif %}">
+                  {% if entry.preview_kind == "image" %}
+                    <img class="entry-preview" src="{{ f|urlencode }}?raw=1" alt="" loading="lazy" decoding="async">
+                  {% elif entry.preview_kind == "text" %}
+                    <pre class="entry-preview" data-url="{{ f|urlencode }}?raw=1" aria-hidden="true">…</pre>
+                  {% else %}
+                    <span class="entry-icon" aria-hidden="true">{% if entry.is_dir %}📁{% else %}📄{% endif %}</span>
+                  {% endif %}
                   <span class="entry-label">
-                    <span class="entry-name">{{ f }}</span>
-                    <span class="entry-kind">{% if f|slice:'-1:' == '/' %}Directory{% else %}File{% endif %}</span>
+                    <span class="entry-name" title="{{ f }}">{{ f }}</span>
                   </span>
                 </a>
-              </li>
+                </td><td class="entry-size">{{ entry.size }}</td><td class="entry-type" title="{{ entry.mimetype }}">{{ entry.mimetype }}</td><td>
+                <a class="entry-download" aria-label="{% blocktranslate %}Download {{ f }}{% endblocktranslate %}" title="{% blocktranslate %}Download {{ f }}{% endblocktranslate %}" href="{{ f|urlencode }}{% if entry.is_dir %}?files=1&amp;download=zip{% else %}?raw=1{% endif %}" download>⬇</a>
+              </td></tr>
+              {% endwith %}
             {% endfor %}
-          </ul>
+            </tbody>
+          </table>
         {% else %}
-          <div class="empty-state">This directory is empty.</div>
+          <div class="empty-state">{% translate "This directory is empty." %}</div>
         {% endif %}
       </section>
     </main>
     <script>
+      document.getElementById("filter-files").addEventListener("input", (event) => {
+        const query = event.target.value.toLowerCase();
+        let count = 0;
+        document.querySelectorAll(".directory-entry:not(.parent)").forEach((row) => {
+          row.hidden = !row.querySelector(".entry-name").textContent.toLowerCase().includes(query);
+          if (!row.hidden) count++;
+        });
+        document.getElementById("visible-file-count").textContent = count;
+      });
+      let sortKey = null, sortDirection = 1;
+      document.querySelectorAll("[data-sort]").forEach((button) => {
+        button.addEventListener("click", () => {
+          sortDirection = sortKey === button.dataset.sort ? -sortDirection : 1;
+          sortKey = button.dataset.sort;
+          const rows = [...document.querySelectorAll(".directory-entry:not(.parent)")];
+          rows.sort((a, b) => {
+            const folders = Number(b.classList.contains("folder")) - Number(a.classList.contains("folder"));
+            if (folders) return folders;
+            const comparison = sortKey === "size"
+              ? Number(a.dataset.size) - Number(b.dataset.size)
+              : a.dataset[sortKey].localeCompare(b.dataset[sortKey], undefined, {numeric: true, sensitivity: "base"});
+            return sortDirection * comparison || a.dataset.name.localeCompare(b.dataset.name, undefined, {numeric: true});
+          });
+          rows.forEach((row) => row.parentElement.appendChild(row));
+          document.querySelectorAll("th[aria-sort]").forEach((header) => header.setAttribute("aria-sort", "none"));
+          button.parentElement.setAttribute("aria-sort", sortDirection === 1 ? "ascending" : "descending");
+        });
+      });
+      document.querySelector('[data-sort="name"]')?.click();
+      const previews = new IntersectionObserver((items) => {
+        for (const item of items) {
+          if (!item.isIntersecting) continue;
+          const preview = item.target;
+          previews.unobserve(preview);
+          fetch(preview.dataset.url)
+            .then((response) => { if (!response.ok) throw new Error("Unavailable"); return response.text(); })
+            .then((text) => { preview.textContent = text.slice(0, 12000); })
+            .catch(() => { preview.textContent = "📄"; });
+        }
+      }, {rootMargin: "100px"});
+      document.querySelectorAll("pre.entry-preview").forEach((preview) => previews.observe(preview));
       window.archiveboxHandleZipClick = function(link, event) {
         if (!link || link.dataset.loading === "1") {
           if (event) event.preventDefault();
@@ -365,7 +185,7 @@
         const spinner = link.querySelector(".archivebox-zip-spinner");
         if (label) {
           link.dataset.originalLabel = label.textContent || "";
-          label.textContent = link.dataset.loadingLabel || "Preparing...";
+          label.textContent = link.dataset.loadingLabel;
           label.style.opacity = "0.92";
         }
         if (spinner) {

archivebox/templates/static/zip_index.html

@@ -0,0 +1,792 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width,initial-scale=1" />
+    <title>Saved files</title>
+    <style>
+      :root {
+        color-scheme: light;
+        --line: #dfe4eb;
+        --muted: #667085;
+        --blue: #245dc1;
+      }
+      * {
+        box-sizing: border-box;
+      }
+      html,
+      body {
+        height: 100%;
+        margin: 0;
+      }
+      body {
+        font: 13px/1.5 system-ui, sans-serif;
+        color: #253247;
+        background: white;
+        display: flex;
+        flex-direction: column;
+      }
+      a {
+        color: var(--blue);
+        text-decoration: none;
+      }
+      button,
+      input {
+        font: inherit;
+      }
+      button,
+      a.action {
+        cursor: pointer;
+        border: 1px solid var(--line);
+        border-radius: 5px;
+        background: white;
+        padding: 5px 10px;
+        color: #344054;
+      }
+      button:hover,
+      a.action:hover {
+        background: #f0f5ff;
+      }
+      header {
+        display: flex;
+        align-items: center;
+        gap: 12px;
+        padding: 10px 16px;
+        border-bottom: 1px solid var(--line);
+      }
+      h1 {
+        font-size: 14px;
+        font-weight: 600;
+        margin: 0;
+        overflow: hidden;
+        text-overflow: ellipsis;
+        white-space: nowrap;
+      }
+      nav {
+        display: flex;
+        gap: 6px;
+        margin-left: auto;
+        flex-shrink: 0;
+      }
+      .toolbar {
+        display: flex;
+        align-items: center;
+        gap: 12px;
+        padding: 9px 16px;
+        background: #f8fafc;
+        border-bottom: 1px solid var(--line);
+      }
+      #breadcrumbs {
+        display: flex;
+        gap: 5px;
+        align-items: center;
+        flex-wrap: wrap;
+        min-width: 0;
+      }
+      #breadcrumbs button {
+        border: 0;
+        background: none;
+        padding: 0;
+        color: var(--blue);
+      }
+      #count {
+        color: var(--muted);
+        white-space: nowrap;
+        margin-left: auto;
+      }
+      #search {
+        width: 160px;
+        border: 1px solid var(--line);
+        border-radius: 5px;
+        padding: 4px 8px;
+      }
+      main {
+        display: flex;
+        min-height: 0;
+        flex: 1;
+      }
+      .listing {
+        overflow: auto;
+        flex: 1;
+        min-width: 260px;
+      }
+      table {
+        width: 100%;
+        border-collapse: collapse;
+        text-align: left;
+      }
+      th {
+        position: sticky;
+        top: 0;
+        background: #fff;
+        color: var(--muted);
+        font-weight: 500;
+        border-bottom: 1px solid var(--line);
+        padding: 8px 16px;
+      }
+      td {
+        border-bottom: 1px solid #eef1f5;
+        padding: 8px 16px;
+      }
+      td.name {
+        width: 100%;
+      }
+      .entry {
+        border: 0;
+        background: none;
+        padding: 0;
+        display: flex;
+        align-items: center;
+        gap: 10px;
+        color: #253247;
+        text-align: left;
+        overflow-wrap: anywhere;
+      }
+      .entry:hover {
+        background: none;
+        color: var(--blue);
+      }
+      .size,
+      .date {
+        color: var(--muted);
+        white-space: nowrap;
+        font-size: 12px;
+      }
+      tr.selected {
+        background: #eaf2ff;
+      }
+      tr:hover {
+        background: #f6f9fc;
+      }
+      #preview {
+        width: 55%;
+        min-width: 280px;
+        border-left: 1px solid var(--line);
+        display: flex;
+        flex-direction: column;
+        min-height: 0;
+      }
+      #preview[hidden] {
+        display: none;
+      }
+      .preview-head {
+        padding: 9px 12px;
+        display: flex;
+        align-items: center;
+        gap: 8px;
+        border-bottom: 1px solid var(--line);
+        background: #f8fafc;
+      }
+      #filename {
+        overflow-wrap: anywhere;
+        font-weight: 600;
+        margin-right: auto;
+      }
+      #content {
+        overflow: auto;
+        padding: 16px;
+        flex: 1;
+        min-height: 0;
+        display: flex;
+        align-items: flex-start;
+        justify-content: center;
+      }
+      #content img {
+        max-width: 100%;
+        max-height: 100%;
+        object-fit: contain;
+      }
+      #content pre {
+        white-space: pre-wrap;
+        overflow-wrap: anywhere;
+        align-self: stretch;
+        margin: 0;
+        width: 100%;
+        font: 12px/1.6 ui-monospace, monospace;
+      }
+      #content iframe {
+        width: 100%;
+        height: 100%;
+        border: 0;
+      }
+      #content audio,
+      #content video {
+        max-width: 100%;
+      }
+      .empty {
+        padding: 22px;
+        color: var(--muted);
+      }
+      #error {
+        color: #b42318;
+        padding: 8px 16px;
+        margin: 0;
+      }
+      #error:empty {
+        display: none;
+      }
+      @media (max-width: 640px) {
+        header {
+          flex-wrap: wrap;
+          padding: 9px 12px;
+        }
+        nav {
+          margin-left: 0;
+        }
+        .toolbar {
+          padding: 8px 12px;
+        }
+        #search {
+          width: 100px;
+        }
+        .date {
+          display: none;
+        }
+        main {
+          flex-direction: column;
+        }
+        .listing {
+          min-height: 140px;
+        }
+        #preview {
+          width: 100%;
+          border-left: 0;
+          border-top: 1px solid var(--line);
+          min-height: 300px;
+          flex: 1;
+        }
+        td,
+        th {
+          padding: 8px 12px;
+        }
+      }
+    </style>
+  </head>
+  <body data-output="{{ output_path }}" data-size="{{ archive_size }}">
+    <header>
+      <h1 id="title">{{ title }}</h1>
+      <nav id="actions" aria-label="Output actions">
+        <a class="action" id="files">View all files</a>
+      </nav>
+    </header>
+    <div class="toolbar">
+      <div id="breadcrumbs"></div>
+      <span id="count"></span
+      ><input
+        id="search"
+        aria-label="Filter files"
+        placeholder="Filter files"
+      />
+    </div>
+    <p id="error" role="alert"></p>
+    <main>
+      <section class="listing" aria-label="Files">
+        <table>
+          <thead>
+            <tr>
+              <th>Name</th>
+              <th>Size</th>
+              <th class="date">Modified</th>
+            </tr>
+          </thead>
+          <tbody id="entries"></tbody>
+        </table>
+      </section>
+      <section id="preview" aria-label="File preview" hidden>
+        <div class="preview-head">
+          <span id="filename"></span><button id="download">Download</button
+          ><button id="close" aria-label="Close preview">×</button>
+        </div>
+        <div id="content"></div>
+      </section>
+    </main>
+    <script>
+      (() => {
+        const byId = (id) => document.getElementById(id),
+          el = (tag, text, cls) => {
+            const n = document.createElement(tag);
+            if (text != null) n.textContent = text;
+            if (cls) n.className = cls;
+            return n;
+          };
+        const output = new URL(document.body.dataset.output, document.baseURI);
+        output.search = "";
+        output.hash = "";
+        const folder = new URL("./", output),
+          raw = new URL(output);
+        raw.searchParams.set("raw", "1");
+        byId("files").href = folder.href + "?files=1";
+        const MAX_PREVIEW = 16 * 1024 * 1024,
+          CHUNK = 256 * 1024,
+          decoder = new TextDecoder();
+        const bytes = (n) =>
+          n < 1024
+            ? n + " B"
+            : n < 1048576
+            ? (n / 1024).toFixed(1) + " KB"
+            : n < 1073741824
+            ? (n / 1048576).toFixed(1) + " MB"
+            : (n / 1073741824).toFixed(2) + " GB";
+        const safePath = (name) => {
+          const p = name.replace(/\\/g, "/").replace(/\/$/, "");
+          return p &&
+            !p.startsWith("/") &&
+            !p.split("/").some((x) => !x || x === "." || x === "..")
+            ? p
+            : null;
+        };
+        const view = (b) => new DataView(b.buffer, b.byteOffset, b.byteLength);
+        const u64 = (v, p) => {
+          const n = v.getBigUint64(p, true);
+          if (n > BigInt(Number.MAX_SAFE_INTEGER))
+            throw Error("ZIP offsets exceed browser precision");
+          return Number(n);
+        };
+        async function rangeResponse(archive, start, length, signal) {
+          if (
+            !Number.isSafeInteger(start) ||
+            !Number.isSafeInteger(length) ||
+            start < 0 ||
+            length <= 0 ||
+            start + length > archive.size
+          )
+            throw Error("Invalid ZIP byte range");
+          const response = await fetch(archive.url, {
+            headers: { Range: `bytes=${start}-${start + length - 1}` },
+            signal,
+          });
+          const expected = `bytes ${start}-${start + length - 1}/${
+            archive.size
+          }`;
+          if (
+            response.status !== 206 ||
+            response.headers.get("Content-Range") !== expected
+          ) {
+            await response.body?.cancel();
+            throw Error("ZIP browsing requires HTTP byte-range support");
+          }
+          return response;
+        }
+        async function range(archive, start, length, signal) {
+          const response = await rangeResponse(archive, start, length, signal),
+            reader = response.body.getReader(),
+            result = new Uint8Array(length);
+          let offset = 0;
+          try {
+            while (true) {
+              const { done, value } = await reader.read();
+              if (done) break;
+              if (offset + value.length > length)
+                throw Error("ZIP range exceeds requested size");
+              result.set(value, offset);
+              offset += value.length;
+            }
+            if (offset !== length) throw Error("Incomplete ZIP byte range");
+            return result;
+          } finally {
+            await reader.cancel();
+          }
+        }
+        async function indexZip(archive) {
+          const tail = await range(
+            archive,
+            Math.max(0, archive.size - 65557),
+            Math.min(archive.size, 65557)
+          );
+          const v = view(tail);
+          let end = -1;
+          for (let i = tail.length - 22; i >= 0; i--)
+            if (
+              v.getUint32(i, true) === 0x06054b50 &&
+              i + 22 + v.getUint16(i + 20, true) === tail.length
+            ) {
+              end = i;
+              break;
+            }
+          if (end < 0) throw Error("Missing ZIP directory");
+          if (v.getUint16(end + 4, true) || v.getUint16(end + 6, true))
+            throw Error("Multi-volume ZIPs are not supported");
+          let count = v.getUint16(end + 10, true),
+            size = v.getUint32(end + 12, true),
+            offset = v.getUint32(end + 16, true);
+          if (count === 65535 || size === 0xffffffff || offset === 0xffffffff) {
+            const locatorOffset = archive.size - tail.length + end - 20,
+              locator = view(await range(archive, locatorOffset, 20));
+            if (
+              locator.getUint32(0, true) !== 0x07064b50 ||
+              locator.getUint32(4, true) !== 0 ||
+              locator.getUint32(16, true) !== 1
+            )
+              throw Error("Invalid ZIP64 locator");
+            const z = view(await range(archive, u64(locator, 8), 56));
+            if (
+              z.getUint32(0, true) !== 0x06064b50 ||
+              z.getUint32(16, true) ||
+              z.getUint32(20, true)
+            )
+              throw Error("Invalid ZIP64 directory");
+            count = u64(z, 32);
+            size = u64(z, 40);
+            offset = u64(z, 48);
+          }
+          if (count > 100000)
+            throw Error(
+              "ZIP has more than 100,000 entries; use the ZIP download"
+            );
+          if (offset + size > archive.size || offset < 0)
+            throw Error("Invalid ZIP directory bounds");
+          let buffer = new Uint8Array(),
+            base = -1;
+          const read = async (position, length) => {
+            if (position < offset || position + length > offset + size)
+              throw Error("Invalid ZIP entry bounds");
+            if (position < base || position + length > base + buffer.length) {
+              base = position;
+              buffer = await range(
+                archive,
+                position,
+                Math.min(Math.max(CHUNK, length), offset + size - position)
+              );
+            }
+            return buffer.subarray(position - base, position - base + length);
+          };
+          const entries = [];
+          let cursor = offset;
+          for (let i = 0; i < count; i++) {
+            const h = view(await read(cursor, 46));
+            if (h.getUint32(0, true) !== 0x02014b50)
+              throw Error("Invalid ZIP directory entry");
+            const nameLength = h.getUint16(28, true),
+              extraLength = h.getUint16(30, true),
+              commentLength = h.getUint16(32, true),
+              flags = h.getUint16(8, true),
+              method = h.getUint16(10, true),
+              crc = h.getUint32(16, true),
+              date = h.getUint16(14, true),
+              time = h.getUint16(12, true);
+            let compressed = h.getUint32(20, true),
+              size = h.getUint32(24, true),
+              local = h.getUint32(42, true);
+            const fields = await read(
+                cursor + 46,
+                nameLength + extraLength + commentLength
+              ),
+              name = decoder.decode(fields.subarray(0, nameLength)),
+              extra = view(
+                fields.subarray(nameLength, nameLength + extraLength)
+              );
+            for (let p = 0; p + 4 <= extra.byteLength; ) {
+              const id = extra.getUint16(p, true),
+                length = extra.getUint16(p + 2, true);
+              if (p + 4 + length > extra.byteLength)
+                throw Error("Invalid ZIP extra field");
+              if (id === 1) {
+                let q = p + 4;
+                const next = () => {
+                  if (q + 8 > p + 4 + length)
+                    throw Error("Invalid ZIP64 entry");
+                  const n = u64(extra, q);
+                  q += 8;
+                  return n;
+                };
+                if (size === 0xffffffff) size = next();
+                if (compressed === 0xffffffff) compressed = next();
+                if (local === 0xffffffff) local = next();
+              }
+              p += 4 + length;
+            }
+            const path = safePath(name);
+            if (path)
+              entries.push({
+                path,
+                directory: name.endsWith("/"),
+                size,
+                compressed,
+                local,
+                flags,
+                method,
+                crc,
+                archive,
+                date: date
+                  ? new Date(
+                      1980 + (date >> 9),
+                      ((date >> 5) & 15) - 1,
+                      date & 31,
+                      time >> 11,
+                      (time >> 5) & 63,
+                      (time & 31) * 2
+                    )
+                  : null,
+              });
+            cursor += 46 + nameLength + extraLength + commentLength;
+          }
+          return entries;
+        }
+        const crcTable = Uint32Array.from({ length: 256 }, (_, n) => {
+          for (let i = 0; i < 8; i++)
+            n = n & 1 ? 0xedb88320 ^ (n >>> 1) : n >>> 1;
+          return n >>> 0;
+        });
+        async function entryStream(file, signal) {
+          if (file.flags & 1)
+            throw Error("Encrypted ZIP entries cannot be previewed");
+          if (![0, 8].includes(file.method))
+            throw Error("This ZIP compression method cannot be previewed");
+          const h = view(await range(file.archive, file.local, 30, signal));
+          if (h.getUint32(0, true) !== 0x04034b50)
+            throw Error("Invalid ZIP file header");
+          const start =
+            file.local + 30 + h.getUint16(26, true) + h.getUint16(28, true);
+          let stream = file.compressed
+            ? (
+                await rangeResponse(
+                  file.archive,
+                  start,
+                  file.compressed,
+                  signal
+                )
+              ).body
+            : new Blob([]).stream();
+          if (file.method === 8)
+            stream = stream.pipeThrough(new DecompressionStream("deflate-raw"));
+          let size = 0,
+            crc = 0xffffffff;
+          return stream.pipeThrough(
+            new TransformStream({
+              transform(chunk, controller) {
+                size += chunk.length;
+                if (size > file.size)
+                  throw Error("ZIP entry exceeds its declared size");
+                for (const byte of chunk)
+                  crc = crcTable[(crc ^ byte) & 255] ^ (crc >>> 8);
+                controller.enqueue(chunk);
+              },
+              flush() {
+                if (size !== file.size || (crc ^ 0xffffffff) >>> 0 !== file.crc)
+                  throw Error("ZIP file size or checksum mismatch");
+              },
+            })
+          );
+        }
+        let files = [],
+          prefix = "",
+          selected = null,
+          controller = null,
+          blobURL = null;
+        const clearPreview = () => {
+          controller?.abort();
+          if (blobURL) URL.revokeObjectURL(blobURL);
+          blobURL = null;
+          byId("content").replaceChildren();
+        };
+        const report = (error) => {
+          if (error.name !== "AbortError")
+            byId("error").textContent = error.message;
+        };
+        function browse(next) {
+          prefix = next;
+          const filter = byId("search").value.toLowerCase(),
+            crumbs = byId("breadcrumbs");
+          crumbs.replaceChildren();
+          const crumb = (label, path) => {
+            const b = el("button", label);
+            b.onclick = () => {
+              byId("search").value = "";
+              browse(path);
+            };
+            crumbs.append(b);
+          };
+          crumb("Files", "");
+          let part = "";
+          for (const segment of prefix.split("/").filter(Boolean)) {
+            part += segment + "/";
+            crumbs.append(el("span", "/"));
+            crumb(segment, part);
+          }
+          const items = new Map();
+          for (const f of files) {
+            if (!f.path.startsWith(prefix)) continue;
+            const tail = f.path.slice(prefix.length),
+              name = tail.split("/")[0];
+            if (!name || !name.toLowerCase().includes(filter)) continue;
+            const directory = f.directory || tail.includes("/");
+            if (!items.has(name) || directory)
+              items.set(
+                name,
+                directory ? { path: prefix + name, directory: true } : f
+              );
+          }
+          const table = byId("entries");
+          table.replaceChildren();
+          for (const file of [...items.values()].sort(
+            (a, b) =>
+              Number(b.directory) - Number(a.directory) ||
+              a.path.localeCompare(b.path)
+          )) {
+            const row = el("tr");
+            if (file === selected) row.className = "selected";
+            const name = el("td", null, "name"),
+              button = el("button", null, "entry");
+            button.append(
+              el("span", file.directory ? "📁" : "📄"),
+              el("span", file.path.slice(prefix.length))
+            );
+            button.onclick = () =>
+              file.directory ? browse(file.path + "/") : openFile(file);
+            name.append(button);
+            row.append(
+              name,
+              el("td", file.directory ? "—" : bytes(file.size), "size"),
+              el("td", file.date?.toLocaleDateString() || "", "date")
+            );
+            table.append(row);
+          }
+          byId("count").textContent = items.size + " items";
+          if (!items.size) {
+            const row = el("tr"),
+              cell = el("td", "No files", "empty");
+            cell.colSpan = 3;
+            row.append(cell);
+            table.append(row);
+          }
+        }
+        const mime = (path) =>
+          ({
+            png: "image/png",
+            jpg: "image/jpeg",
+            jpeg: "image/jpeg",
+            gif: "image/gif",
+            webp: "image/webp",
+            avif: "image/avif",
+            pdf: "application/pdf",
+            mp3: "audio/mpeg",
+            ogg: "audio/ogg",
+            wav: "audio/wav",
+            mp4: "video/mp4",
+            webm: "video/webm",
+          }[path.split(".").pop().toLowerCase()] || "application/octet-stream");
+        async function openFile(file) {
+          clearPreview();
+          selected = file;
+          controller = new AbortController();
+          const signal = controller.signal;
+          byId("error").textContent = "";
+          byId("preview").hidden = false;
+          byId("filename").textContent = file.path.split("/").pop();
+          browse(prefix);
+          const content = byId("content");
+          content.append(el("span", "Loading…", "empty"));
+          if (file.size > MAX_PREVIEW) {
+            content.replaceChildren(
+              el("span", "File is too large to preview. Use Download.", "empty")
+            );
+            return;
+          }
+          try {
+            const stream = file.archive
+              ? await entryStream(file, signal)
+              : (await fetch(file.url, { signal })).body;
+            const blob = await new Response(stream).blob();
+            if (signal.aborted) return;
+            const type = mime(file.path);
+            content.replaceChildren();
+            if (type !== "application/octet-stream") {
+              blobURL = URL.createObjectURL(new Blob([blob], { type }));
+              let node;
+              if (type.startsWith("image/")) {
+                node = el("img");
+                node.alt = file.path;
+              } else if (type === "application/pdf") {
+                node = el("iframe");
+                node.title = file.path;
+                node.setAttribute("sandbox", "");
+              } else {
+                node = el(type.startsWith("audio/") ? "audio" : "video");
+                node.controls = true;
+              }
+              node.src = blobURL;
+              content.append(node);
+            } else if (
+              /\.(txt|md|csv|tsv|json|xml|html?|css|js|py|log|ya?ml|svg|rtf)$/i.test(
+                file.path
+              )
+            )
+              content.append(el("pre", await blob.text()));
+            else
+              content.append(
+                el(
+                  "span",
+                  "Preview unavailable for this file type. Use Download.",
+                  "empty"
+                )
+              );
+          } catch (error) {
+            if (!signal.aborted) {
+              content.replaceChildren();
+              report(error);
+            }
+          }
+        }
+        byId("search").oninput = () => browse(prefix);
+        byId("close").onclick = () => {
+          clearPreview();
+          selected = null;
+          byId("preview").hidden = true;
+          browse(prefix);
+        };
+        byId("download").onclick = async () => {
+          if (!selected) return;
+          const file = selected;
+          byId("error").textContent = "";
+          if (!file.archive) {
+            const a = el("a");
+            a.href = file.url;
+            a.download = file.path;
+            a.click();
+            return;
+          }
+          try {
+            // Stream large entries to disk where supported; never buffer an entire ZIP.
+            if ("showSaveFilePicker" in window) {
+              const handle = await window.showSaveFilePicker({
+                suggestedName: file.path.split("/").pop(),
+              });
+              const writable = await handle.createWritable();
+              try {
+                await (await entryStream(file)).pipeTo(writable);
+              } catch (e) {
+                await writable.abort().catch(() => {});
+                throw e;
+              }
+            } else {
+              if (file.size > MAX_PREVIEW)
+                throw Error(
+                  "Large individual downloads require a browser with streaming file saves. Use ZIP to download the archive."
+                );
+              const blob = await new Response(await entryStream(file)).blob(),
+                url = URL.createObjectURL(blob),
+                a = el("a");
+              a.href = url;
+              a.download = file.path.split("/").pop();
+              a.click();
+              setTimeout(() => URL.revokeObjectURL(url), 60000);
+            }
+          } catch (error) {
+            report(error);
+          }
+        };
+        const download = el("a", "Download ZIP", "action");
+        download.href = raw.href;
+        download.download = "";
+        byId("actions").insertBefore(download, byId("files"));
+        document.title = byId("title").textContent;
+        indexZip({ url: raw.href, size: Number(document.body.dataset.size) })
+          .then((entries) => {
+            files = entries;
+            browse("");
+          })
+          .catch(report);
+      })();
+    </script>
+  </body>
+</html>

archivebox/tests/conftest.py

@@ -14,6 +14,7 @@ import textwrap
 import time
 import shutil
 import ctypes
+from urllib.parse import urlsplit
 from datetime import timedelta
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
@@ -22,6 +23,13 @@ from types import SimpleNamespace
 from typing import Any
 from collections.abc import Callable
 
+from abx_plugins.plugins.opencode.archivebox.conftest import (
+    opencode_archive_config as opencode_archive_config,
+    installed_opencode as installed_opencode,
+    live_opencode as live_opencode,
+    agent_server as agent_server,
+)
+
 import psutil
 import pytest
 import requests
@@ -32,6 +40,14 @@ PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve()
 SESSION_DATA_DIR = Path(
     os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"),
 ).resolve()
+PORT_ALLOCATION_DIR = Path(os.environ.setdefault("ARCHIVEBOX_PYTEST_PORTS_DIR", str(SESSION_DATA_DIR / "ports")))
+if worker_id := os.environ.get("PYTEST_XDIST_WORKER"):
+    # Workers inherit the coordinator's environment. Give each one its own
+    # collection and cleanup root so a finishing worker cannot delete another's.
+    SESSION_DATA_DIR /= worker_id
+    for runtime_key in ("ABXPKG_LIB_DIR", "PERSONAS_DIR", "CHROME_USER_DATA_DIR"):
+        if runtime_path := os.environ.get(runtime_key):
+            os.environ[runtime_key] = str(Path(runtime_path) / worker_id)
 
 os.environ["ARCHIVEBOX_PYTEST_SESSION_DATA_DIR"] = str(SESSION_DATA_DIR)
 os.environ["DATA_DIR"] = str(SESSION_DATA_DIR)
@@ -323,6 +339,18 @@ def pytest_configure():
     if not apps.ready:
         django.setup()
 
+    if os.environ.get("PYTEST_XDIST_WORKER"):
+        from django.conf import settings
+        from archivebox.config import django as django_config
+        from archivebox.config.common import get_config
+
+        # pytest-django may import startup config before this conftest isolates
+        # the worker's paths. Refresh that real startup config once so HTTP
+        # requests resolve the same installed binaries as worker subprocesses;
+        # otherwise a service restart searches the coordinator's empty lib dir.
+        django_config.CONFIG = get_config(include_machine=False)
+        settings.CONFIG = django_config.CONFIG
+
 
 @pytest.fixture(scope="session")
 def django_db_modify_db_settings(
@@ -437,7 +465,7 @@ def hermetic_lib_dir(tmp_path):
 @pytest.fixture
 def cached_abxpkg_lib_dir():
     """Reuse one real abxpkg installation cache for the current pytest session."""
-    lib_dir = SESSION_DATA_DIR / "lib"
+    lib_dir = Path(os.environ["ABXPKG_LIB_DIR"]) if os.environ.get("ABXPKG_LIB_DIR") else SESSION_DATA_DIR / "lib"
     lib_dir.mkdir(parents=True, exist_ok=True)
     original_lib_dir = os.environ.get("ABXPKG_LIB_DIR")
     os.environ["ABXPKG_LIB_DIR"] = str(lib_dir)
@@ -1076,6 +1104,8 @@ def start_archivebox_server(
     else:
         output = log_path.read_text(encoding="utf-8", errors="replace") if log_path else ""
         raise AssertionError(f"ArchiveBox server did not open port {port} within 30 seconds:\n{output}")
+    response = get_http_response(port, host=f"127.0.0.1:{port}", path="/health/", timeout=5)
+    assert response.status_code == 200 and response.headers.get("X-ArchiveBox-Health") == "OK", response.text
     return proc
 
 
@@ -1294,6 +1324,43 @@ def api_auth_headers(api_token: str, *, django_client: bool = False, port: int |
     }
 
 
+class LocalhostHTTPAdapter(requests.adapters.HTTPAdapter):
+    """Connect real HTTP requests to loopback while preserving URL authorities."""
+
+    def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None):
+        url = urlsplit(request.url)
+        if url.scheme != "http" or not (url.hostname == "localhost" or (url.hostname or "").endswith(".localhost")):
+            raise ValueError(f"Expected an HTTP localhost test URL, got {request.url}")
+        return self.poolmanager.connection_from_host("127.0.0.1", port=url.port, scheme="http")
+
+    def send(self, request, **kwargs):
+        # Send Host only on the wire. Requests copies prepared headers on
+        # redirects and uses Host for cookie matching; leaving it attached
+        # would select the previous origin's cookies after a cross-host redirect.
+        original_host = request.headers.get("Host")
+        request.headers["Host"] = urlsplit(request.url).netloc
+        try:
+            return super().send(request, **kwargs)
+        finally:
+            if original_host is None:
+                request.headers.pop("Host", None)
+            else:
+                request.headers["Host"] = original_host
+
+
+def localhost_session() -> requests.Session:
+    # Browsers resolve *.localhost themselves; Linux libc in stock containers
+    # often does not. This is curl --connect-to for real server tests: only the
+    # socket destination changes. Keep the original URL so Requests exercises
+    # real redirect and cookie-domain rules, including dynamic snap-* hosts.
+    # Never replace URLs with 127.0.0.1 or stub responses: that hides isolation
+    # bugs. This also avoids global /etc/hosts edits and runner-specific DNS.
+    session = requests.Session()
+    session.trust_env = False
+    session.mount("http://", LocalhostHTTPAdapter())
+    return session
+
+
 def wait_for_live_api(port: int, *, path: str = "/api/v1/docs"):
     return get_http_response(port, host=f"api.archivebox.localhost:{port}", path=path)
 
@@ -1339,9 +1406,20 @@ def init_archive(cwd: Path) -> None:
 
 
 def get_free_port() -> int:
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        sock.bind(("127.0.0.1", 0))
-        return sock.getsockname()[1]
+    # Binding port zero only proves availability now. Another pytest worker can
+    # receive that same port before either service starts. Claim it atomically
+    # for this session while the socket is still bound, across all workers and
+    # helper subprocesses; the session cleanup removes these small markers.
+    PORT_ALLOCATION_DIR.mkdir(parents=True, exist_ok=True)
+    while True:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+            sock.bind(("127.0.0.1", 0))
+            port = sock.getsockname()[1]
+            try:
+                (PORT_ALLOCATION_DIR / str(port)).touch(exist_ok=False)
+            except FileExistsError:
+                continue
+            return port
 
 
 def stop_server(cwd: Path) -> None:
@@ -1663,6 +1741,8 @@ def resolve_abxpkg_binary_env(
     deps_from: Path | list[Path] | tuple[Path, ...] | None = None,
 ) -> dict[str, str]:
     """Resolve required test dependencies through abxpkg's normal install path."""
+    from abxpkg.config import apply_exec_env
+
     command_env = dict(env) if env is not None else os.environ.copy()
     command_env["ABXPKG_LIB_DIR"] = str(lib_dir)
     command = [
@@ -1683,7 +1763,11 @@ def resolve_abxpkg_binary_env(
     )
     assert result.returncode == 0, result.stderr or result.stdout
     payload = json.loads(result.stdout)
-    return {str(key): str(value) for key, value in payload.items()}
+    # abxpkg env emits prepend/append deltas, not a replacement environment.
+    # Applying a PATH ending in ':' verbatim hides the caller's abxpkg CLI.
+    delta = {str(key): str(value) for key, value in payload.items()}
+    apply_exec_env(delta, command_env)
+    return {key: command_env[key] for key in delta}
 
 
 def run_test_hook(
@@ -1781,7 +1865,7 @@ def install_real_binary(
     from archivebox.machine.models import Binary, Machine
 
     binary = Binary.objects.create(
-        machine=machine or Machine.current(refresh=True),
+        machine=machine or Machine.current(),
         name=name,
         binproviders=binproviders,
         overrides=overrides or {},
@@ -1931,3 +2015,60 @@ def create_test_snapshot_json(url: str | None = None, **kwargs) -> dict[str, Any
         "status": kwargs.get("status", "queued"),
         **{k: v for k, v in kwargs.items() if k not in ("tags_str", "status")},
     }
+
+
+@pytest.fixture
+def browser_runtime(cached_abxpkg_lib_dir):
+    from abx_plugins import get_plugins_dir
+    from abx_plugins.plugins.base.utils import get_config
+
+    # UI tests need isolated collections, but share the browser installation.
+    # Resolve it through the same abxpkg schema/cache as capture hooks rather
+    # than downloading Playwright's separate Chromium in CI. This keeps local
+    # and CI runs on the supported install path without per-test cold installs.
+    shared_lib = cached_abxpkg_lib_dir
+    env = cli_env(
+        ABXPKG_LIB_DIR=str(shared_lib),
+        CHROME_HEADLESS="True",
+        CHROME_SANDBOX="False",
+        CHROME_ISOLATION="snapshot",
+    )
+    env.pop("CHROME_BINARY", None)
+    resolved_env = resolve_abxpkg_chrome_env(shared_lib, env)
+    chrome_config = get_config(
+        Path(get_plugins_dir()) / "chrome" / "config.json",
+        environ={**env, **resolved_env},
+        hydrate_binaries=False,
+    )
+
+    return {
+        "lib_dir": shared_lib,
+        "node_modules_dir": Path(resolved_env["NODE_MODULES_DIR"]),
+        "node_path": resolved_env["NODE_PATH"],
+        "node_binary": Path(resolved_env["NODE_BINARY"]),
+        "chrome_binary": Path(resolved_env["CHROME_BINARY"]),
+        "chrome_args": [*chrome_config.CHROME_ARGS, *chrome_config.CHROME_ARGS_EXTRA],
+    }
+
+
+def _reset_runtime_config() -> None:
+    from archivebox.config import common
+    from archivebox.machine.models import Machine
+
+    for value in vars(common).values():
+        cache_clear = getattr(value, "cache_clear", None)
+        if cache_clear is not None:
+            cache_clear()
+    Machine.current()
+
+
+def _set_archivebox_config(data_dir: Path, *values: str, env: dict[str, str] | None = None) -> None:
+    os.chdir(data_dir)
+    result = run_archivebox_cmd(
+        ["config", "--set", *values],
+        cwd=data_dir,
+        env=env,
+        timeout=120,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+    _reset_runtime_config()

archivebox/tests/fixtures/responsive-images/README.md

@@ -0,0 +1,13 @@
+# Responsive-image replay regression
+
+`google-drive.wacz` contains two unmodified, gzip-compressed WARC response
+records from a real ArchiveWeb.page capture on 2026-10-04:
+
+- https://drive.google.com/drive/folders/1KpLl_1tcK0eeehzN980zbG-3M2nhbVks
+- https://www.gstatic.com/images/branding/productlogos/drive_2026/v1/web-48dp/logo_drive_2026_color_1x_web_48dp.png
+
+The public folder's HTML declares both 1x and 2x logo URLs, but the recording
+only contains the 1x image. These records were copied byte-for-byte from the
+capture's `archive/data.warc.gz` using its CDX offsets; the subset CDX offsets and datapackage hashes were recomputed. No HTTP
+responses were synthesized or edited. Other resources were omitted to keep the fixture small.
+The test replays this actual archive at DPR 1 and 2, entirely from local storage.

archivebox/tests/test_api_discovery_performance.py

@@ -0,0 +1,54 @@
+"""Server discovery must not repeat schema generation or probe installed tools."""
+
+import sys
+import threading
+import os
+from pathlib import Path
+
+import pytest
+from django.test.utils import override_script_prefix
+
+from archivebox.config import VERSION
+from archivebox.machine.models import Machine
+from archivebox.tests.conftest import install_real_binary, resolve_abxpkg_binary_env
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("script_name", ["", "/mounted"])
+def test_repeated_discovery_reuses_schema_without_binary_filesystem_probes(client, hermetic_lib_dir, script_name):
+    install_real_binary("node", binproviders="env,apt,brew")
+    resolve_abxpkg_binary_env(hermetic_lib_dir, "node")
+    machine = Machine.current()
+    machine.config = {**machine.config, "NODE_BINARY": str(hermetic_lib_dir / "env" / "bin" / "node")}
+    machine.save(update_fields=["config"])
+    request = {"HTTP_HOST": "api.archivebox.localhost:5797", "SCRIPT_NAME": script_name}
+    with override_script_prefix(script_name):
+        first = client.get("/api/v1/openapi.json", **request)
+    assert first.status_code == 200, first.content
+    schema_calls = []
+    filesystem_calls = []
+
+    def observe(frame, event, arg):
+        if event == "call" and frame.f_code.co_name == "model_json_schema":
+            schema_calls.append(frame.f_code.co_filename)
+        if (event == "call" and frame.f_code.co_name in {"stat", "lstat", "open", "iterdir", "glob", "rglob"}) or (
+            event == "c_call" and arg in (os.stat, os.lstat)
+        ):
+            path = frame.f_locals.get("self", frame.f_locals.get("path"))
+            if isinstance(path, (str, Path)) and Path(path).is_relative_to(hermetic_lib_dir):
+                filesystem_calls.append(str(path))
+
+    previous = sys.getprofile()
+    previous_thread = threading.getprofile()
+    threading.setprofile_all_threads(observe)
+    try:
+        with override_script_prefix(script_name):
+            second = client.get("/api/v1/openapi.json", **request)
+    finally:
+        threading.setprofile_all_threads(previous_thread)
+        sys.setprofile(previous)
+    assert second.status_code == 200, second.content
+    assert second.json() == first.json()
+    assert second.json()["info"]["version"] == VERSION
+    assert f"{script_name}/api/v1/core/snapshots" in second.json()["paths"]
+    assert (len(schema_calls), filesystem_calls) == (0, [])

archivebox/tests/test_api_v1_cli_add.py

@@ -478,3 +478,40 @@ def test_api_cli_add_rejects_file_path_and_shell_injection_payloads(tmp_path):
     with use_archivebox_db(tmp_path):
         tag_names = set(SnapshotTag.objects.filter(snapshot=snapshot).values_list("tag__name", flat=True))
     assert "api-security" in tag_names
+
+
+def test_api_add_only_queues_work_without_storage_or_worker_control(client, api_headers):
+    import sys
+    from archivebox.config import CONSTANTS
+
+    calls = []
+
+    def observe(frame, event, arg):
+        if event != "call":
+            return
+        name = frame.f_code.co_name
+        if name == "ensure_background_runner":
+            calls.append(name)
+        if name in {"stat", "lstat", "exists", "is_dir", "is_file", "is_symlink", "mkdir", "iterdir", "symlink_to"}:
+            path = frame.f_locals.get("self")
+            if isinstance(path, Path) and path.is_relative_to(CONSTANTS.ARCHIVE_DIR):
+                calls.append((name, str(path)))
+
+    previous = sys.getprofile()
+    sys.setprofile(observe)
+    try:
+        response = client.post(
+            "/api/v1/cli/add",
+            data={"urls": ["https://example.com/queue-only"], "tag": "queue-only"},
+            content_type="application/json",
+            **api_headers,
+        )
+    finally:
+        sys.setprofile(previous)
+    assert response.status_code == 200, response.content
+    crawl = Crawl.objects.get(pk=response.json()["result"]["crawl_id"])
+    assert crawl.urls == "https://example.com/queue-only"
+    assert crawl.status == Crawl.StatusChoices.QUEUED
+    assert crawl.retry_at is not None
+    assert not crawl.snapshot_set.exists()
+    assert calls == []

archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py

@@ -477,25 +477,38 @@ def test_paused_snapshot_survives_server_restart_and_resumes_via_api(client, tmp
         stop_server(tmp_path)
 
 
-def test_rest_snapshot_delete_removes_output_dir(client, api_headers):
-    url = "https://example.com/delete-path-snapshot"
+@pytest.mark.timeout(180)
+def test_rest_snapshot_delete_removes_output_dir(tmp_path, recursive_test_site):
+    import time
+    from .test_snapshot_service import _snapshot_state
+    from .conftest import create_admin_and_token, live_api_request, start_archivebox_server, stop_archivebox_process
 
-    response = api_client_request(
-        client,
-        "post",
-        "/api/v1/core/snapshots",
-        payload={"url": url, "depth": 0, "status": Snapshot.StatusChoices.QUEUED},
-        headers=api_headers,
-    )
-    assert response.status_code == 200, response.content.decode()
-
-    snapshot = Snapshot.objects.get(url=url)
-    snapshot_dir = Path(snapshot.output_dir)
-    snapshot_dir.mkdir(parents=True, exist_ok=True)
-    (snapshot_dir / "delete-path-test.txt").write_text("snapshot output")
-    assert snapshot_dir.exists()
-
-    response = client.delete(f"/api/v1/core/snapshot/{snapshot.id}", **api_headers)
-    assert response.status_code == 200, response.content.decode()
-    assert not Snapshot.objects.filter(pk=snapshot.pk).exists()
-    assert not snapshot_dir.exists()
+    init_archive(tmp_path)
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget")
+    url = recursive_test_site["root_url"]
+    captured = run_archivebox_cmd(["add", "--plugins=wget", url], cwd=tmp_path, env=env)
+    assert captured.returncode == 0, captured.stdout + captured.stderr
+    state = _snapshot_state(tmp_path, url)
+    assert any("Root" in path.read_text(errors="ignore") for path in (state["snapshot_dir"] / "wget").rglob("*.html"))
+    with use_archivebox_db(tmp_path):
+        snapshot_id = str(Snapshot.objects.get(url=url).pk)
+    token = create_admin_and_token(tmp_path)
+    server = start_archivebox_server(tmp_path, port=port, env=env, log_name="api-delete-handoff.log")
+    try:
+        response = live_api_request(port, "delete", f"/api/v1/core/snapshot/{snapshot_id}", api_token=token)
+        assert response.status_code == 200, response.text
+        assert response.json()["queued_count"] == 1
+        assert response.json()["deleted_count"] == 0
+        deadline = time.monotonic() + 75
+        with use_archivebox_db(tmp_path):
+            while Snapshot.objects.filter(pk=snapshot_id).exists() and time.monotonic() < deadline:
+                assert server.poll() is None
+                time.sleep(0.1)
+            assert not Snapshot.objects.filter(pk=snapshot_id).exists()
+        assert not state["snapshot_dir"].exists()
+        assert not state["crawl_link"].is_symlink()
+        missing = live_api_request(port, "get", f"/api/v1/core/snapshot/{snapshot_id}", api_token=token)
+        assert missing.status_code == 404
+    finally:
+        stop_archivebox_process(server)

archivebox/tests/test_api_v1_core_snapshots.py

@@ -13,6 +13,65 @@ from archivebox.crawls.models import Crawl
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+def test_browser_metadata_respects_only_new_and_reports_capture_owner(client, api_headers):
+    url = "https://example.com/browser-deduplication"
+    for name in ("Original", "Replacement"):
+        persona = client.post(
+            "/api/v1/personas/sync",
+            data={"extension_persona_id": f"dedupe-{name}", "name": name},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert persona.status_code == 200, persona.content
+
+    def submit(only_new, persona):
+        queued = client.post(
+            "/api/v1/cli/add",
+            data={"urls": [url], "only_new": only_new, "persona": persona},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert queued.status_code == 200, queued.content
+        crawl_id = queued.json()["result"]["crawl_id"]
+        metadata = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": crawl_id},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert metadata.status_code == 200, metadata.content
+        return crawl_id, metadata.json()
+
+    first_crawl, first = submit(True, "Original")
+    second_crawl, reused = submit(True, "Replacement")
+    assert second_crawl != first_crawl
+    assert reused["id"] == first["id"]
+    assert reused["crawl_id"] == first_crawl
+    assert reused["persona"] == "Original"
+    assert Snapshot.objects.filter(url=url).count() == 1
+    forced_crawl, forced = submit(False, "Replacement")
+    assert forced["id"] != first["id"]
+    assert forced["crawl_id"] == forced_crawl
+    assert forced["persona"] == "Replacement"
+    assert Snapshot.objects.filter(url=url).count() == 2
+
+
+def test_snapshot_delete_only_queues_cleanup(client, api_headers):
+    created = client.post(
+        "/api/v1/core/snapshots",
+        data={"url": "https://example.com/queued-delete"},
+        content_type="application/json",
+        **api_headers,
+    )
+    assert created.status_code == 200, created.content
+    snapshot_id = created.json()["id"]
+    response = client.delete(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+    assert response.status_code == 200, response.content
+    assert response.json()["queued_count"] == 1
+    assert response.json()["deleted_count"] == 0
+    assert Snapshot.objects.get(pk=snapshot_id).status == Snapshot.DELETING_STATE
+
+
 def test_snapshots_api_filters_status_column(client, api_admin_user, api_headers):
     crawl = Crawl.objects.create(
         urls="https://example.com",
@@ -150,3 +209,81 @@ def test_new_snapshot_creation_does_not_wait_for_active_crawl(client, api_admin_
     assert response.status_code == 200, response.content
     assert Snapshot.objects.filter(url=url, crawl=crawl).count() == 1
     assert elapsed < 1
+
+
+def test_snapshot_metadata_requests_do_not_touch_archive_storage(client, api_admin_user, api_headers):
+    """Metadata writes and serialization must work without replay storage access."""
+    import sys
+    import threading
+    from pathlib import Path
+    from archivebox.config import CONSTANTS
+
+    url = "https://example.com/metadata-without-storage"
+    crawl = Crawl.objects.create(urls=url, created_by=api_admin_user)
+    storage_calls = []
+
+    def observe(frame, event, arg):
+        if event == "call" and frame.f_code.co_name in {
+            "stat",
+            "lstat",
+            "exists",
+            "is_dir",
+            "is_file",
+            "is_symlink",
+            "open",
+            "mkdir",
+            "iterdir",
+            "glob",
+            "rglob",
+            "readlink",
+            "symlink_to",
+            "unlink",
+        }:
+            path = frame.f_locals.get("self", frame.f_locals.get("path"))
+            if isinstance(path, (str, Path)) and Path(path).is_relative_to(CONSTANTS.ARCHIVE_DIR):
+                storage_calls.append((frame.f_code.co_name, str(path)))
+
+    previous = sys.getprofile()
+    thread_profile = threading.getprofile()
+    threading.setprofile_all_threads(observe)
+    try:
+        response = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": str(crawl.id), "title": "Browser title", "tags": ["browser"]},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert response.status_code == 200, response.content
+        snapshot_id = response.json()["id"]
+        updated = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": str(crawl.id), "title": "Updated title", "tags": ["browser"]},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert updated.status_code == 200, updated.content
+        result = client.post(
+            "/api/v1/core/archiveresults",
+            data={"snapshot_id": snapshot_id, "plugin": "chrome_extension_viewport", "status": "started"},
+            **api_headers,
+        )
+        assert result.status_code == 200, result.content
+        result_id = result.json()["id"]
+        result_get = client.get(f"/api/v1/core/archiveresult/{result_id}", **api_headers)
+        assert result_get.status_code == 200, result_get.content
+        assert result_get.json()["output_files"] == {}
+        fetched = client.get(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+        assert fetched.status_code == 200, fetched.content
+        assert fetched.json()["archiveresults"][0]["id"] == result_id
+        deleted = client.delete(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+        assert deleted.status_code == 200, deleted.content
+        assert deleted.json()["queued_count"] == 1
+    finally:
+        threading.setprofile_all_threads(thread_profile)
+        sys.setprofile(previous)
+    snapshot = Snapshot.objects.get(pk=snapshot_id)
+    assert snapshot.title == "Updated title"
+    assert snapshot.status == Snapshot.DELETING_STATE
+    assert list(snapshot.tags.values_list("name", flat=True)) == ["browser"]
+    assert fetched.json()["title"] == "Updated title"
+    assert storage_calls == []

archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py

@@ -1,4 +1,6 @@
 import pytest
+import os
+import time
 
 from archivebox.crawls.models import Crawl
 
@@ -15,3 +17,19 @@ def test_basic_success_case_request(client, tmp_path, api_admin_user, api_header
     response = client.get(f"/api/v1/crawls/crawl/{crawl.id}/files/folder/basic.txt", **api_headers)
 
     assert response.status_code == 200, response.content
+
+
+def test_screencast_frame_is_checked_by_image_request(client, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com/screencast", created_by=api_admin_user)
+    frame = crawl.output_dir / "chrome_screencast" / "latest.jpg"
+    frame.parent.mkdir(parents=True, exist_ok=True)
+    frame.write_bytes(b"stored frame")
+    started_at = time.time()
+    os.utime(frame, (started_at - 10, started_at - 10))
+    url = f"/api/v1/crawls/crawl/{crawl.id}/files/chrome_screencast/latest.jpg?after={started_at}"
+    assert client.get(url, **api_headers).status_code == 404
+    os.utime(frame, (started_at + 1, started_at + 1))
+    response = client.get(url, **api_headers)
+    assert response.status_code == 200
+    assert b"".join(response.streaming_content) == b"stored frame"
+    assert client.get(url.replace(str(started_at), "invalid"), **api_headers).status_code == 400

archivebox/tests/test_api_v1_machine_binaries.py

@@ -8,7 +8,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 
 def test_basic_success_case_request(client, tmp_path, api_headers):
-    machine = Machine.current(refresh=True)
+    machine = Machine.current()
     binary = install_real_binary("python3", machine=machine)
 
     response = client.get("/api/v1/machine/binaries", **api_headers)

archivebox/tests/test_api_v1_machine_binary_binary_id.py

@@ -8,7 +8,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 
 def test_basic_success_case_request(client, tmp_path, api_headers):
-    machine = Machine.current(refresh=True)
+    machine = Machine.current()
     binary = install_real_binary("python3", machine=machine)
 
     response = client.get(f"/api/v1/machine/binary/{binary.id}", **api_headers)

archivebox/tests/test_api_v1_machine_binary_by_name_name.py

@@ -8,7 +8,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 
 def test_basic_success_case_request(client, tmp_path, api_headers):
-    machine = Machine.current(refresh=True)
+    machine = Machine.current()
     binary = install_real_binary("python3", machine=machine)
 
     response = client.get("/api/v1/machine/binary/by-name/python3", **api_headers)