Bump release version to 0.9.57

# Conflicts:
#	pyproject.toml
#	uv.lock

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -30,8 +30,8 @@ body:
       description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
       render: markdown
       placeholder: |
-        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox` in iTerm2
-        2. Went to the https://127.0.0.1:8000/add/ page in Google Chrome
+        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 5797:5797 archivebox/archivebox` in iTerm2
+        2. Went to the https://127.0.0.1:5797/add/ page in Google Chrome
         3. Typed 'https://example.com/xyz' into the 'Add URL' input field
         4. Clicked the 'Add+' button
         5. Got a 500 error and saw the errors below in terminal
@@ -138,7 +138,7 @@ body:
             archivebox:
                 image: archivebox/archivebox:latest
                 ports:
-                    - 8000:8000
+                    - 5797:5797
                 volumes:
                     - ./data:/data
                 environment:

.github/configs/ci-tooling.json

@@ -116,6 +116,10 @@
     }
   ],
   "docker_binaries": [
+    {
+      "name": "{BASH_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
     {
       "name": "{DOCKER_BINARY}",
       "binproviders": "env"

.github/pages/README.md

@@ -0,0 +1,119 @@
+# archivebox.io website
+
+This repository owns its complete website. Build and deploy it without checking
+out another repository's website, fetching shared snippets, or calling a shared
+workflow. Common files are ordinary committed copies; update them deliberately.
+
+## Repository-specific files
+
+- Content: `publicsite/index.html`.
+- Site/gallery generation: `bin/generate_ui_screenshot_gallery.py`.
+- Application capture: `bin/collect_ui_screenshots.sh`.
+- Deployment branch: `dev`.
+
+## Common publication contract
+
+1. Build pull requests without deploying them.
+2. Build website pushes immediately, independently of application capture CI.
+3. Restore validated screenshots from trusted default-branch capture jobs.
+   Keep the original capture revision, timestamp, coverage and image checksums.
+4. Retain `site-screenshots` artifacts for 90 days. After expiry, restore the
+   published image set. Never replace a complete gallery with an empty one when
+   restoration fails. Apple device groups are independently validated artifacts;
+   Android's checked-in bootstrap captures retain their local provenance.
+5. Render current content and layout around those captures. Site revision and
+   capture revision are separate facts; reusing captures must not relabel them.
+6. Serialize production restore/build/deploy under one concurrency group and
+   check out the latest default-branch revision when each build starts. Keep the
+   built revision in provenance; later release-bot commits must not block deploys.
+7. Keep platform-specific build, installation, capture, and completeness checks
+   in the application's existing capture workflow. Site-only edits do not need
+   to rerun those jobs.
+
+## Common presentation contract
+
+The base header, Apps menu and icons, footer, brand styles and navigation behavior
+should remain identical copies across the seven product sites. Put this site's
+name, navigation, CTA and extra footer resources outside the common base files.
+Preserve real links, anchor IDs, no-JavaScript navigation, keyboard operation,
+mobile layout, reduced-motion preferences and existing product-specific content.
+Screenshots stay in English. Translation content, when added, lives in separate
+HTML files; no runtime translation service or cross-repository dependency.
+
+## Editing and building
+
+`base/` contains identical independent copies across the seven product sites.
+Common Python scripts share the same behavior; repository formatters may change
+line wrapping without changing their syntax trees. The base owns the shared logo, Apps menu/icons, footer directory, navigation
+behavior and styles. `site.json` and `nav.html` contain this site's name, routes,
+links and CTA. Product content and additional footer resources stay in the native
+HTML/Jinja/Liquid templates listed above. The `ARCHIVEBOX:*` comments are build
+slots; the renderer fills them with static HTML, never browser-side fetches.
+
+The existing builder calls `site.py render` after generating its content. This
+copies `base/chrome.css` and `base/chrome.js` to `site-base/`, assembles the common
+HTML, validates local resources and writes the site revision to root `build.json`.
+Screenshot manifests are left untouched. No build reads another repository.
+
+Common changes are manual copy/paste edits. Compare `base/` and `site.py` with
+another product repository when updating them; do not add a synchronization job,
+package dependency or reusable cross-repository workflow. `nav.html`, `site.json`
+and the marked workflow build section are intentionally repository-specific.
+
+Build: `uv run --no-project python .github/pages/site.py render _site --source publicsite` after restoring screenshots and refreshing the gallery.
+
+Check the built site with real Chromium (also run by Pages CI):
+
+```sh
+uv run --no-config --no-project --with playwright==1.63.0 playwright install chromium
+uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py SITE_OUTPUT --evidence /tmp/site-evidence
+```
+
+Use this workflow's `SITE_OUTPUT` directory. The check covers desktop/mobile
+layout, keyboard dismissal, no-JavaScript links, local resources and visible
+images, and uploads its screenshots as `site-verification` for 14 days.
+
+## Branding and future translations
+
+`base/chrome.css` owns the shared system font stacks, plum accent and CTA shape,
+including the existing `.button` and `.ab-btn` classes. Product styles own their
+layouts, feature sections and platform-specific button states. Shared system
+fonts need no external font downloads. `site.py` emits the same complete SEO/OG/
+Twitter tags everywhere while retaining each native template's title, description,
+canonical URL, locale and social artwork; edit those values with that page's copy.
+
+For translations, keep complete separate HTML files under `es/`, `fr/` and `zh/`,
+and list their output paths in `site.json` alongside the English pages. Share CSS,
+JS, social artwork and the English screenshot gallery. Translate each page's title
+and description, set its own canonical and `lang` (`zh` for Chinese), and add
+reciprocal `hreflang` links only for translations that actually exist. The renderer
+preserves these tags and computes shared asset paths for nested pages. Localized
+header/footer copy and the small language selector can be added with those pages;
+there is no language redirect, placeholder translation or translation runtime now.
+
+## Screenshot strips
+
+`marquee.py` and `base/marquee.css` / `base/marquee.js` are also identical copies.
+Product homepages choose their placement with an `ARCHIVEBOX:MARQUEE` comment.
+The renderer takes real images, captions, dimensions and links from the generated
+`screenshots/index.html`; there is no second screenshot list or browser fetch.
+Desktop/tablet/mobile galleries contribute their desktop views. Apple uses
+`MARQUEE-CLIENT` and `MARQUEE-SERVER` slots to keep its two products separate.
+Catalog sites without a slot do not load the strip assets.
+
+The strip scrolls once, pauses on hover, focus or manual interaction, and has an
+explicit Play/Pause button. Reduced-motion users start paused. Without JavaScript,
+normal image links and horizontal scrolling remain available. Screenshot capture
+manifests and provenance are never rewritten by this presentation step.
+
+Standalone one- or two-image illustrations remain in their authored source.
+Gallery strips with three or more images use the latest restored captures.
+
+## Keeping the common layout in sync
+
+The comments in `base/header.html`, `base/footer-start.html`,
+`base/footer-end.html`, `base/chrome.css`, and `base/chrome.js` list the peer
+repositories. Update those committed copies together when shared branding,
+Apps links, keyboard behavior, or footer resources change. Preserve each site's
+local navigation and content. The former monorepo `site-chrome` generator is
+retired; builds must use only this repository's files.

.github/pages/artifacts.py

@@ -0,0 +1,102 @@
+"""Common screenshot artifact transport; coverage checks belong to each app."""
+
+import json
+import subprocess
+from pathlib import PurePosixPath
+from urllib.request import urlopen
+
+
+def api(repo, path):
+    return json.loads(subprocess.check_output(["gh", "api", f"repos/{repo}/{path}"]))
+
+
+def trusted(run, repo, workflow, branch, successful=True):
+    return (
+        (run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
+        and run["head_branch"] == branch
+        and run["event"] in ("push", "workflow_dispatch")
+        and run["path"] == f".github/workflows/{workflow}"
+        and (not successful or run["conclusion"] == "success")
+    )
+
+
+def runs(repo, workflow, branch, successful=True, artifact_names=("site-screenshots",)):
+    # Ask for capture artifacts directly instead of checking every historical
+    # workflow run (most app runs never produce a complete screenshot set).
+    # Artifact creation time also includes fresh captures from old rerun jobs.
+    candidates = []
+    for name in artifact_names:
+        rows = subprocess.check_output(
+            [
+                "gh",
+                "api",
+                f"repos/{repo}/actions/artifacts?name={name}&per_page=100",
+                "--paginate",
+                "--jq",
+                ".artifacts[] | select(.expired == false) | @json",
+            ],
+            text=True,
+        )
+        candidates.extend(json.loads(row) for row in rows.splitlines())
+    seen = set()
+    for artifact in sorted(candidates, key=lambda item: item["created_at"], reverse=True):
+        source = artifact.get("workflow_run") or {}
+        run_id = source.get("id")
+        if (
+            artifact["name"] not in artifact_names
+            or not run_id
+            or run_id in seen
+            or source.get("head_branch") != branch
+            or source.get("head_repository_id") != source.get("repository_id")
+        ):
+            continue
+        seen.add(run_id)
+        run = api(repo, f"actions/runs/{run_id}")
+        if trusted(run, repo, workflow, branch, successful):
+            yield run
+
+
+def names(repo, run):
+    rows = subprocess.check_output(
+        [
+            "gh",
+            "api",
+            f"repos/{repo}/actions/runs/{run['id']}/artifacts?per_page=100",
+            "--paginate",
+            "--jq",
+            ".artifacts[] | @json",
+        ],
+        text=True,
+    )
+    return {artifact["name"] for row in rows.splitlines() if not (artifact := json.loads(row))["expired"]}
+
+
+def download(repo, run, name, destination):
+    subprocess.run(
+        [
+            "gh",
+            "run",
+            "download",
+            str(run["id"]),
+            "--repo",
+            repo,
+            "--name",
+            name,
+            "--dir",
+            str(destination),
+        ],
+        check=True,
+    )
+
+
+def relative_path(name):
+    path = PurePosixPath(name)
+    if not name or path.is_absolute() or ".." in path.parts or any(c in name for c in ":\\?#%"):
+        raise ValueError(f"Unsafe screenshot path: {name}")
+    return path
+
+
+def fetch(base, name):
+    relative_path(name)
+    with urlopen(base + name, timeout=60) as response:
+        return response.read()

.github/pages/base/chrome.css

@@ -0,0 +1,156 @@
+/*
+Shared ArchiveBox website navigation styles and behavior.
+Keep this file in sync with .github/pages/base/chrome.css in:
+ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
+electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
+Each site builds its committed copy independently; there is no central generator.
+Keep site-specific navigation, CTAs, and extra footer links outside this base.
+Also review the inline navbar/footer in abx-dl/website/index.html and
+ debian-archivebox/website/index.html when shared links or branding change.
+*/
+/* Common brand tokens. Native templates own product layout and content. */
+:root {
+  --abx-font-sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+  --abx-font-mono: ui-monospace, "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace;
+  --abx-accent: #9b2854;
+  --abx-accent-hover: #782040;
+  --abx-paper: #fcfaf7;
+  --abx-ink: #28252a;
+  --abx-muted: #716c74;
+  --abx-line: #e9e2e3;
+  --font-sans: var(--abx-font-sans);
+  --font-mono: var(--abx-font-mono);
+  --ab-font-sans: var(--abx-font-sans);
+  --ab-font-serif: var(--abx-font-sans);
+  --ab-font-label: var(--abx-font-sans);
+  --ab-font-mono: var(--abx-font-mono);
+  --ab-accent: var(--abx-accent);
+  --ab-accent-hover: var(--abx-accent-hover);
+  --ab-on-accent: #fff;
+}
+body { font-family: var(--abx-font-sans); }
+code, pre, kbd, samp { font-family: var(--abx-font-mono); }
+/* Reuse existing CTA classes; platform selection continues to control colors. */
+.button, .ab-btn {
+  display: inline-flex; align-items: center; justify-content: center; gap: 10px;
+  min-height: 44px; height: auto; max-width: 100%; padding: 10px 20px;
+  border-radius: 24px; font-family: var(--abx-font-sans);
+  font-size: 14px; font-weight: 650; line-height: 1.4;
+  text-align: center; white-space: normal; text-decoration: none;
+}
+.button.primary, .ab-btn--primary { background: var(--abx-accent); border-color: var(--abx-accent); color: #fff; }
+.button.primary:hover, .ab-btn--primary:hover { background: var(--abx-accent-hover); border-color: var(--abx-accent-hover); color: #fff; }
+.button:focus-visible, .ab-btn:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
+.button svg, .button img, .ab-btn svg, .ab-btn img { flex-shrink: 0; }
+@media (prefers-reduced-motion: reduce) { .button, .ab-btn { transition: none; } .button:hover { transform: none; } }
+
+/* Shared ArchiveBox site chrome. Independent copy; edit alongside the other product sites, without a build dependency. */
+/* Native anchor navigation leaves room for the heading and its surrounding content. */
+html { scroll-behavior: smooth; scroll-padding-top: clamp(96px, 25vh, 240px); }
+@media (prefers-reduced-motion: reduce) { html { scroll-behavior: auto; } }
+.abx-header, .abx-footer { box-sizing: border-box; font-family: var(--abx-font-sans); font-size: 14px; line-height: 1.5; letter-spacing: normal; text-align: left; }
+.abx-header *, .abx-footer * { box-sizing: border-box; }
+.abx-header { position: sticky; top: 0; z-index: 100; display: flex; align-items: center; justify-content: space-between; gap: 24px; width: 100%; max-width: none; margin: 0; padding: 15px max(24px, calc((100% - 1280px) / 2)); border: 0; border-bottom: 1px solid #e9e2e3; background: #fcfaf7f5; color: #28252a; backdrop-filter: blur(16px); }
+.abx-header a, .abx-footer a { text-decoration: none; text-underline-offset: .24em; }
+.abx-header .abx-brand, .abx-footer .abx-brand { display: inline-flex; flex: 0 0 auto; align-items: center; gap: 11px; margin: 0; padding: 0; color: inherit; font-size: 19px; font-weight: 750; line-height: 1.2; letter-spacing: -.5px; white-space: nowrap; }
+.abx-brand .abx-logo { display: block; flex: 0 0 auto; width: 32px; height: 32px; margin: 0; }
+.abx-header .abx-subsite { transform: translateY(3px); margin-left: 2px; color: #716c74; font-size: 12px; font-weight: 450; letter-spacing: 0; }
+.abx-header .abx-nav { display: flex; align-items: center; justify-content: flex-end; flex-wrap: wrap; gap: 0; width: auto; max-width: none; margin: 0; padding: 0; font-size: 13px; font-weight: 600; }
+.abx-nav > a, .abx-nav > details { margin: 0 0 0 15px; padding: 0 0 0 15px; border: 0; border-left: 1px solid #ded4d8; border-radius: 0; background: none; }
+.abx-nav > :first-child { margin-left: 0; padding-left: 0; border-left: 0; }
+.abx-header .abx-nav a { display: inline-flex; align-items: center; min-height: 36px; color: #4d424b; white-space: nowrap; }
+.abx-header .abx-nav a:hover { color: #9b2854; }
+.abx-header .abx-nav .abx-cta { margin-left: 18px; padding: 9px 17px; border: 1px solid #9b2854; border-radius: 24px; background: #9b2854; color: #fff; }
+.abx-header .abx-nav .abx-cta:hover { background: #782040; color: #fff; }
+.abx-header .abx-apps { position: relative; }
+.abx-apps > summary { display: flex; align-items: center; gap: 7px; min-height: 36px; list-style: none; cursor: pointer; color: #4d424b; }
+.abx-apps > summary::-webkit-details-marker { display: none; }
+.abx-apps > summary::before { content: '›'; display: inline-block; color: #9b2854; font-size: 22px; line-height: 1; transition: transform .15s; }
+.abx-apps[open] > summary::before { transform: rotate(90deg); }
+.abx-header .abx-app-links { position: absolute; right: 0; top: calc(100% + 12px); width: 440px; max-width: calc(100vw - 24px); max-height: min(70vh, 580px); overflow-y: auto; padding: 8px; border: 1px solid #e9e2e3; border-radius: 16px; background: #fcfaf7; box-shadow: 0 18px 48px #2d152522, 0 3px 8px #2d152508; }
+.abx-header .abx-app-row { display: flex; align-items: center; gap: 8px; min-height: 44px; padding: 4px 6px; border-radius: 10px; transition: background .15s; }
+.abx-header .abx-app-row:hover, .abx-header .abx-app-row:focus-within { background: #f2e8ed; }
+.abx-header .abx-app-links .abx-app-name { flex: 1; min-width: 0; min-height: 34px; padding: 5px 4px; border-radius: 6px; font-size: 13px; line-height: 1.35; white-space: normal; }
+.abx-header .abx-platforms { display: flex; flex: 0 0 auto; align-items: center; gap: 5px; }
+.abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 auto; height: 30px; min-height: 30px; padding: 5px; border: 1px solid #e8dfe3; border-radius: 9px; background: #fffdfb; color: #4d424b; box-shadow: 0 2px 3px #2d152510, inset 0 1px 0 #fff; transition: transform .15s, box-shadow .15s, border-color .15s; }
+.abx-header .abx-app-links .abx-platform { width: 30px; }
+.abx-header .abx-platform svg { display: block; width: 18px; height: 18px; margin: 0; fill: currentColor; }
+.abx-header .abx-app-links .abx-platform-docker { color: #2386ce; }
+.abx-header .abx-app-links .abx-platform-windows, .abx-header .abx-app-links .abx-platform-app-store, .abx-header .abx-app-links .abx-platform-safari { color: #1879c4; }
+.abx-header .abx-app-links .abx-platform-android { color: #29965d; }
+.abx-header .abx-app-links .abx-platform-firefox { color: #e46b24; }
+.abx-header .abx-app-links .abx-platform-brave { color: #e8552b; }
+.abx-header .abx-app-links .abx-platform-edge { color: #168b95; }
+.abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-platform:focus-visible, .abx-header .abx-app-links .abx-tool:hover, .abx-header .abx-app-links .abx-tool:focus-visible { transform: translateY(-2px); border-color: #c99bb0; background: #fff; box-shadow: 0 5px 10px #2d15251c; }
+.abx-header .abx-app-links .abx-tool { padding: 5px 8px; font-size: 11px; white-space: nowrap; }
+.abx-header .abx-tools-row { margin-top: 5px; padding-top: 9px; border-top: 1px solid #e9e2e3; border-radius: 0 0 10px 10px; }
+.abx-header .abx-tools-row .abx-app-name { flex: 0 0 auto; }
+.abx-header .abx-tools-row .abx-platforms { flex: 0 1 auto; flex-wrap: wrap; justify-content: flex-end; margin-left: auto; }
+.abx-header .abx-app-links code { display: inline; vertical-align: baseline; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-size: 1em; font-weight: inherit; line-height: inherit; margin: 0; padding: 0; border: 0; background: transparent; color: inherit; }
+.abx-header .abx-icon-stack { isolation: isolate; gap: 0; padding: 4px 5px; }
+.abx-header .abx-icon-stack .abx-integration { position: relative; z-index: 1; transform: translateY(var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); transition: transform .18s ease, box-shadow .18s ease, border-color .18s ease; }
+.abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -10px; }
+.abx-header .abx-icon-stack .abx-integration:nth-child(2) { --stack-angle: 9deg; --stack-y: -2px; z-index: 2; color: #569a31; }
+.abx-header .abx-icon-stack .abx-integration:nth-child(3) { --stack-angle: -6deg; --stack-y: 2px; z-index: 3; }
+.abx-header .abx-icon-stack .abx-integration:nth-child(4) { --stack-angle: 7deg; --stack-y: -1px; z-index: 4; color: #f00; }
+.abx-header .abx-icon-stack .abx-integration:nth-child(5) { --stack-angle: -10deg; --stack-y: 1px; z-index: 5; }
+.abx-header .abx-icon-stack .abx-integration:nth-child(6) { --stack-angle: 6deg; --stack-y: -2px; z-index: 6; }
+.abx-header .abx-icon-stack .abx-integration:nth-child(7) { --stack-angle: -5deg; z-index: 7; }
+.abx-header .abx-icon-stack .abx-integration:not(:hover, :focus-visible):has(~ .abx-integration:is(:hover, :focus-visible)) { transform: translate(-6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
+.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) ~ .abx-integration:not(:hover, :focus-visible) { transform: translate(6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
+.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { z-index: 20; transform: translateY(-7px) rotate(0deg) scale(1.12); border-color: #c99bb0; box-shadow: 0 8px 14px #2d15252b; }
+@media (hover: none) { .abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -6px; } }
+@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration { transition: none; } }
+.abx-footer { position: relative; z-index: 1; display: block; width: 100%; max-width: none; margin: 72px 0 0; padding: 60px max(24px, calc((100% - 1280px) / 2)) 24px; border: 0; border-top: 3px solid #7e284a; background: radial-gradient(ellipse at 12% 0, #54243b55, transparent 48%), linear-gradient(120deg, #201923, #14151b 75%); color: #ddd2dd; color-scheme: dark; }
+.abx-footer .abx-footer-intro { display: flex; align-items: flex-start; justify-content: space-between; gap: 32px; margin: 0 0 42px; }
+.abx-footer .abx-brand { color: #fff4ed; font-size: 24px; }
+.abx-footer .abx-brand .abx-logo { width: 40px; height: 40px; }
+.abx-footer .abx-tagline { max-width: 430px; margin: 12px 0 0; color: #b8acbb; font-size: 14px; line-height: 1.7; }
+.abx-footer .abx-footer-motto { margin: 2px 0 0; color: #d8b39f; font: italic 25px/1.45 Georgia, serif; text-align: right; }
+.abx-footer .abx-footer-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 42px; margin: 0; }
+.abx-footer .abx-footer-column { margin: 0; padding: 20px 0 0; border: 0; border-top: 1px solid #67505e66; background: none; }
+.abx-footer .abx-footer-column h2 { margin: 0 0 19px; padding: 0; border: 0; color: #e4acbb; font: 650 12px/1.4 var(--abx-font-sans); letter-spacing: .12em; text-transform: uppercase; }
+.abx-footer .abx-footer-column ul { display: block; list-style: none; margin: 0; padding: 0; }
+.abx-footer .abx-footer-column li { display: flex; align-items: center; gap: 10px; margin: 0; padding: 0; min-height: 36px; }
+.abx-footer .abx-footer-column li::before { content: none; }
+.abx-footer .abx-footer-column a { color: #ddd2dd; font-size: 13px; line-height: 1.55; }
+.abx-footer .abx-footer-column a:first-child { padding: 6px 0; }
+.abx-footer a:hover { color: #fff; text-decoration: underline; }
+.abx-footer .abx-source { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 28px; width: 28px; height: 28px; border: 1px solid transparent; border-radius: 50%; color: #a58d9f; }
+.abx-footer .abx-source:hover { color: #f3b5cb; border-color: #80516a; background: #ffffff08; }
+.abx-source svg { width: 15px; height: 15px; fill: currentColor; }
+.abx-footer .abx-footer-bottom { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px 24px; margin-top: 32px; padding-top: 21px; border-top: 1px solid #67505e66; color: #a99bab; font-size: 12px; }
+.abx-footer .abx-footer-bottom p { margin: 0; font-size: inherit; color: inherit; }
+.abx-footer .abx-footer-bottom a { color: #cebbc9; }
+.abx-footer .abx-footer-local { margin: 24px 0 0; padding: 16px 0 0; border: 0; border-top: 1px solid #67505e66; border-radius: 0; background: none; color: #baacbc; font-size: 12px; }
+.abx-footer-local > summary { cursor: pointer; color: #d6bfce; }
+.abx-footer-local > div { padding-top: 18px; }
+.abx-footer-local a { color: #e5aabd; }
+.abx-footer-local .brand, .abx-footer-local .footer-intro, .abx-footer-local .footer-grid, .abx-footer-local .footer-meta { color: inherit; }
+.abx-footer-local .footer-card { background: transparent; border-color: #67505e66; }
+.abx-footer-local h2, .abx-footer-local h3, .abx-footer-local strong { color: #e4d8e3; }
+.abx-footer-local .footer-link-copy span, .abx-footer-local .footer-card-subtitle { color: #baacbc; }
+.abx-header a:focus-visible, .abx-header summary:focus-visible, .abx-footer a:focus-visible, .abx-footer summary:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
+@media (max-width: 1100px) {
+  .abx-header { flex-wrap: wrap; gap: 8px 24px; padding: 12px 20px; }
+  .abx-header .abx-nav { justify-content: flex-start; }
+  .abx-header .abx-app-links { position: fixed; top: auto; left: 20px; right: auto; width: 440px; margin-top: 10px; }
+}
+@media (max-width: 700px) {
+  .abx-header { position: relative; padding: 14px 18px; }
+  .abx-header .abx-brand { flex-wrap: wrap; row-gap: 5px; }
+  .abx-header .abx-nav { width: 100%; gap: 4px 0; font-size: 12px; }
+  .abx-nav > a, .abx-nav > details { margin-left: 10px; padding-left: 10px; }
+  .abx-header .abx-nav .abx-cta { margin-left: 10px; padding: 7px 13px; }
+  .abx-header .abx-app-links { left: 12px; right: 12px; width: auto; max-height: 60vh; }
+  .abx-footer { padding: 40px 22px 22px; margin-top: 48px; }
+  .abx-footer .abx-footer-intro { display: block; margin-bottom: 30px; }
+  .abx-footer .abx-footer-motto { margin-top: 20px; text-align: left; font-size: 22px; }
+  .abx-footer .abx-footer-grid { grid-template-columns: 1fr; gap: 26px; }
+  .abx-footer .abx-footer-column li { min-height: 40px; }
+}
+@media (prefers-reduced-motion: reduce) { .abx-apps > summary::before, .abx-header .abx-app-row, .abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { transition: none; } .abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-tool:hover { transform: none; } }
+
+.abx-footer-local > div > a + a { margin-left: 12px; }
+
+@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { transform: none; } }

.github/pages/base/chrome.js

@@ -0,0 +1,28 @@
+/*
+Shared ArchiveBox website navigation styles and behavior.
+Keep this file in sync with .github/pages/base/chrome.js in:
+ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
+electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
+Each site builds its committed copy independently; there is no central generator.
+Keep site-specific navigation, CTAs, and extra footer links outside this base.
+Also review the inline navbar/footer in abx-dl/website/index.html and
+ debian-archivebox/website/index.html when shared links or branding change.
+*/
+(() => {
+  const menu = document.querySelector('.abx-apps');
+  document.addEventListener('keydown', event => {
+    if (event.key === 'Escape' && menu?.open) { menu.open = false; menu.querySelector('summary').focus(); }
+  });
+  document.addEventListener('click', event => { if (menu?.open && !menu.contains(event.target)) menu.open = false; });
+  function revealAnchor() {
+    let target;
+    try { target = document.getElementById(decodeURIComponent(window.location.hash.slice(1))); } catch { return; }
+    if (!target) return;
+    let details = target.closest('details');
+    let opened = false;
+    while (details) { if (!details.open) { details.open = true; opened = true; } details = details.parentElement.closest('details'); }
+    if (opened) window.requestAnimationFrame(() => target.scrollIntoView());
+  }
+  window.addEventListener('hashchange', revealAnchor);
+  revealAnchor();
+})();

.github/pages/base/footer-end.html

@@ -0,0 +1,12 @@
+<!--
+Shared ArchiveBox website footer.
+Keep this file in sync with .github/pages/base/footer-end.html in:
+ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
+electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
+Each site builds its committed copy independently; there is no central generator.
+Keep site-specific navigation, CTAs, and extra footer links outside this base.
+Also review the inline navbar/footer in abx-dl/website/index.html and
+ debian-archivebox/website/index.html when shared links or branding change.
+-->
+  <div class="abx-footer-bottom"><p>Built in the open. Supported by our community.</p><p><a href="https://github.com/ArchiveBox">GitHub organization</a> · <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Security &amp; privacy</a> · <a href="https://archivebox.io">ArchiveBox.io ↗</a></p></div>
+</footer>

.github/pages/base/footer-start.html

@@ -0,0 +1,13 @@
+<!--
+Shared ArchiveBox website footer.
+Keep this file in sync with .github/pages/base/footer-start.html in:
+ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
+electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
+Each site builds its committed copy independently; there is no central generator.
+Keep site-specific navigation, CTAs, and extra footer links outside this base.
+Also review the inline navbar/footer in abx-dl/website/index.html and
+ debian-archivebox/website/index.html when shared links or branding change.
+-->
+<footer class="abx-footer" aria-label="ArchiveBox ecosystem">
+  <div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
+  <div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps &amp; tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS &amp; iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS &amp; iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated

.github/pages/base/header.html

@@ -0,0 +1,24 @@
+<!--
+Shared ArchiveBox website navbar and Apps menu.
+Keep this file in sync with .github/pages/base/header.html in:
+ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
+electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
+Each site builds its committed copy independently; there is no central generator.
+Keep site-specific navigation, CTAs, and extra footer links outside this base.
+Also review the inline navbar/footer in abx-dl/website/index.html and
+ debian-archivebox/website/index.html when shared links or branding change.
+-->
+<header class="abx-header">
+  <a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span><span class="abx-subsite">__SITE_NAME__</span></a>
+  <nav class="abx-nav" aria-label="Main navigation">
+    <details class="abx-apps"><summary>Apps</summary><div class="abx-app-links">
+      <div class="abx-app-row"><a class="abx-app-name" href="https://archivebox.io/#install-compose">ArchiveBox Server</a><div class="abx-platforms"><a class="abx-platform abx-platform-docker" href="https://archivebox.io/#install-compose" aria-label="ArchiveBox Server for Docker" title="ArchiveBox Server for Docker"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M349.9 236.3l-66.1 0 0-59.4 66.1 0 0 59.4zm0-204.3l-66.1 0 0 60.7 66.1 0 0-60.7zm78.2 144.8l-66.1 0 0 59.4 66.1 0 0-59.4zM271.8 104.7l-66.1 0 0 60.1 66.1 0 0-60.1zm78.1 0l-66.1 0 0 60.1 66.1 0 0-60.1zm276.8 100c-14.4-9.7-47.6-13.2-73.1-8.4-3.3-24-16.7-44.9-41.1-63.7l-14-9.3-9.3 14c-18.4 27.8-23.4 73.6-3.7 103.8-8.7 4.7-25.8 11.1-48.4 10.7l-434.7 0c-8.7 50.8 5.8 116.8 44 162.1 37.1 43.9 92.7 66.2 165.4 66.2 157.4 0 273.9-72.5 328.4-204.2 21.4 .4 67.6 .1 91.3-45.2 1.5-2.5 6.6-13.2 8.5-17.1l-13.3-8.9zM115.6 176.8l-66 0 0 59.4 66.1 0 0-59.4-.1 0zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm-78.1-72.1l-66.1 0 0 60.1 66.1 0 0-60.1z"/></svg></a><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/#archivebox-serverapp" aria-label="ArchiveBox Server for macOS" title="ArchiveBox Server for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://electron.archivebox.io/" aria-label="ArchiveBox Server for Windows" title="ArchiveBox Server for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://archivebox.io/#install-apt" aria-label="ArchiveBox Server for Linux" title="ArchiveBox Server for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-1Line truncated
+      <div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Desktop</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Desktop for macOS" title="ArchiveBox Desktop for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://windows.archivebox.io/" aria-label="ArchiveBox Desktop for Windows" title="ArchiveBox Desktop for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://linux.archivebox.io/" aria-label="ArchiveBox Desktop for Linux" title="ArchiveBox Desktop for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-10 14.1-10.5 4.7-4.4 13.5-14.2 27.9-14.2 7.1 0 15.6 2.3 25.9 8.9 6.3 4.1 11.3 4.4 22.6 9.3 8.4 3.5 13.7 9.7 10.5 18.2-2.6 7.1-11 14.4-22.7 18.1-11.1 3.6-19.8 16-38.2 14.9-3.9-.2-7-1-9.6-2.1-8-3.5-12.2-10.4-20-15-8.6-4.8-13.2-10.4-14.7-15.3-1.4-4.9 0-9 4.2-12.3zm3.3 334c-2.7 35.1-43.9 34.4-75.3 18-29.9-15.8-68.6-6.5-76.5-21.9-2.4-4.7-2.4-12.7 2.6-26.4l0-.2c2.4-7.6 .6-16-.6-23.9-1.2-7.8-1.8-15 .9-20 3.5-6.7 8.5-9.1 14.8-11.3 10.3-3.7 11.8-3.4 19.6-9.9 5.5-5.7 9.5-12.9 14.3-18 5.1-5.5 10-8.1 17.7-6.9 8.1 1.2 15.1 6.8 21.9 16l19.6 35.6c9.5 19.9 43.1 48.4 41 68.9zm-1.4-25.9c-4.1-6.6-9.6-13.6-14.4-19.6 7.1 0 14.2-2.2 16.7-8.9 2.3-6.2 0-14.9-7.4-24.9-13.5-18.2-38.3-32.5-38.3-32.5-13.5-8.4-21.1-18.7-24.6-29.9s-3-23.3-.3-35.2c5.2-22.9 18.6-45.2 27.2-59.2 2.3-1.7 .8 3.2-8.7 20.8-8.5 16.1-24.4 53.3-2.6 82.4 .6-20.7 5.5-41.8 13.8-61.5 12-27.4 37.3-74.9 39.3-112.7 1.1 .8 4.6 3.2 6.2 4.1 4.6 2.7 8.1 6.7 12.6 10.3 12.4 10 28.5 9.2 42.4 1.2 6.2-3.5 11.2-7.5 15.9-9 9.9-3.1 17.8-8.6 22.3-15 7.7 30.4 25.7 74.3 37.2 95.7 6.1 11.4 18.3 35.5 23.6 64.6 3.3-.1 7 .4 10.9 1.4 13.8-35.7-11.7-74.2-23.3-84.9-4.7-4.6-4.9-6.6-2.6-6.5 12.6 11.2 29.2 33.7 35.2 59 2.8 11.6 3.3 23.7 .4 35.7 16.4 6.8 35.9 17.9 30.7 34.8-2.2-.1-3.2 0-4Line truncated
+      <div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Mobile</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Mobile for iOS" title="ArchiveBox Mobile for iOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-android" href="https://android.archivebox.io/" aria-label="ArchiveBox Mobile for Android" title="ArchiveBox Mobile for Android"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!-- Font Awesome Free 5.0.13 by Fonticons, Inc. https://fontawesome.com License: https://creativecommons.org/licenses/by/4.0/ --><path d="M89.6 204.5v115.8c0 15.4-12.1 27.7-27.5 27.7-15.3 0-30.1-12.4-30.1-27.7V204.5c0-15.1 14.8-27.5 30.1-27.5 15.1 0 27.5 12.4 27.5 27.5zm10.8 157c0 16.4 13.2 29.6 29.6 29.6h19.9l.3 61.1c0 36.9 55.2 36.6 55.2 0v-61.1h37.2v61.1c0 36.7 55.5 36.8 55.5 0v-61.1h20.2c16.2 0 29.4-13.2 29.4-29.6V182.1H100.4v179.4zm248-189.1H99.3c0-42.8 25.6-80 63.6-99.4l-19.1-35.3c-2.8-4.9 4.3-8 6.7-3.8l19.4 35.6c34.9-15.5 75-14.7 108.3 0L297.5 34c2.5-4.3 9.5-1.1 6.7 3.8L285.1 73c37.7 19.4 63.3 56.6 63.3 99.4zm-170.7-55.5c0-5.7-4.6-10.5-10.5-10.5-5.7 0-10.2 4.8-10.2 10.5s4.6 10.5 10.2 10.5c5.9 0 10.5-4.8 10.5-10.5zm113.4 0c0-5.7-4.6-10.5-10.2-10.5-5.9 0-10.5 4.8-10.5 10.5s4.6 10.5 10.5 10.5c5.6 0 10.2-4.8 10.2-10.5zm94.8 60.1c-15.1 0-27.5 12.1-27.5 27.5v115.8c0 15.4 12.4 27.7 27.5 27.7 15.4 0 30.1-12.4 30.1-27.7V204.5c0-15.4-14.8-27.5-30.1-27.5z"/></svg></a></div></div>
+      <div class="abx-app-row"><a class="abx-app-name" href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><div class="abx-platforms"><a class="abx-platform abx-platform-chrome" href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj" aria-label="ArchiveBox Browser Extension for Chrome" title="ArchiveBox Browser Extension for Chrome"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc. --><path d="M0 256C0 209.4 12.47 165.6 34.27 127.1L144.1 318.3C166 357.5 207.9 384 256 384C270.3 384 283.1 381.7 296.8 377.4L220.5 509.6C95.9 492.3 0 385.3 0 256zM365.1 321.6C377.4 302.4 384 279.1 384 256C384 217.8 367.2 183.5 340.7 160H493.4C505.4 189.6 512 222.1 512 256C512 397.4 397.4 511.1 256 512L365.1 321.6zM477.8 128H256C193.1 128 142.3 172.1 130.5 230.7L54.19 98.47C101 38.53 174 0 256 0C350.8 0 433.5 51.48 477.8 128V128zM168 256C168 207.4 207.4 168 256 168C304.6 168 344 207.4 344 256C344 304.6 304.6 344 256 344C207.4 344 168 304.6 168 256z"/></svg></a><a class="abx-platform abx-platform-safari" href="https://extension.archivebox.io/" aria-label="ArchiveBox Browser Extension for Safari" title="ArchiveBox Browser Extension for Safari"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M274.7 274.7L237.3 237.3 166 346 274.7 274.7zM256 8a248 248 0 1 0 0 496 248 248 0 1 0 0-496zM411.8 182.8l14.8-6.1c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.8 1 2.6 1.7 1.3 1.6 1.7 2.6 .6 2 .6 3.1-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.3-2.6 1.7L418 197.6c-2 .8-4.2 .8-6.1 0s-3.5-2.4-4.3-4.3-.8-4.2 0-6.1 2.4-3.5 4.3-4.3zM314.4 94l6.1-14.8c.8-2 2.4-3.5 4.3-4.3s4.2-.8 6.1 0 3.5 2.4 4.3 4.3 .8 4.2 0 6.1l-6.1 14.8c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0c-1-.4-1.9-1-2.6-1.7s-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1zM256 60c2.1 0 4.2 .8 5.7 2.3S264 65.9 264 68l0 16c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7S253.9 60 256 60zM181 74.9c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.9 1 2.6 1.7 1.3 1.6 1.7 2.6L197.6 94c.4 1 .7 2 .7 3.1s-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.4-2.6 1.8-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.8-1.3-1.7-1.7-2.7l-6.1-14.8c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3zm-63.6 42.5c1.5-1.5 3.5-2.3 5.7-2.3s4.2 .8 5.7 2.3L140 128.7c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3l-11.3-11.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6l0 0zM60 256c0-2.1 .8-4.2 2.3-5.7S65.9 248 68 248l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3l-16 0c-2.1 0-4.2-.8-5.7-2.3S60 258.1 60 256zm40.2 73.2l-14.8 6.1c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1 1-1.9 1.7-2.6 1.6-1.3 2.6-1.7L94 314.4c2-.8 4.2-.8 6.1 0s3.5 2.4 4.3 4.3 .8 4.2 0 6.1-2.4 3.5-4.3 4.3zm4.3-136c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1 2.4-3.5 4.3-4.3 4.2-.8 6.1 0l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1l0 0zM197.6 418l-6.1 14.8c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1l6.1-14.8c.4-1 1-1.9 1.7-2.7s1.6-1.4 2.6-1.8 2-.6 3.1-.6 2.1 .2 3.1 .6 1.9 1 2.6 1.8 1.3 1.7 1.7 2.6 .6 2 .6 3.1-.2 2.1-.7 3.1zM264 444c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7s3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3 2.3 3.5 2.3 5.7l0 16zm67-6.9c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6L314.4 418c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3 4.2-.8 6.1 0 3.5 2.4 4.3 4.3l6.1 14.8c.8 2 .8 4.2 0 6.1s-2.4 3.5-4.3 4.3zm63.6-42.5c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3L372 383.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6 3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3l11.3 11.3c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6l0 0zM286.2 286.2L110.3 401.7 225.8 225.8 401.7 110.3 286.2 286.2zM437.1 331c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1c.4-1 1-1.9 1.7-2.6s1.6-1.3 2.6-1.7 2-.6 3.1-.6 2.1 .2 3.1 .6l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1zm6.9-67l-16 0c-2.1 0-4.2-.8-5.7-2.3s-2.3-3.5-2.3-5.7 .8-4.2 2.3-5.7 3.5-2.3 5.7-2.3l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3z"/></svg></a><a class="abx-platform abx-platform-firefox" href="https://addons.mozilla.org/firefox/addon/archivebox-exporter/" aria-label="ArchiveBox Browser Extension for Firefox" title="ArchiveBox Browser Extension for Firefox"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512Line truncated
+      <div class="abx-app-row abx-plugin-row"><a class="abx-app-name" href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><div class="abx-platforms abx-icon-stack"><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Google Drive storage" title="Google Drive storage"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path d="m38.7 419.3 22.6 39c4.7 8.2 11.4 14.7 19.4 19.4l80.6-139.6H0c0 9.1 2.3 18.2 7 26.4z" style="fill:#0066da"/><path d="M256 173.9 175.4 34.3c-7.9 4.7-14.7 11.1-19.4 19.4L7 311.7c-4.6 8-7 17.1-7 26.4h161.3z" style="fill:#00ac47"/><path d="M431.4 477.7c7.9-4.7 14.7-11.1 19.4-19.4l9.4-16.1 44.9-77.7c4.7-8.2 7-17.3 7-26.4H350.7l34.3 67.4z" style="fill:#ea4335"/><path d="m256 173.9 80.6-139.6c-7.9-4.7-17-7-26.4-7H201.8c-9.4 0-18.5 2.6-26.4 7z" style="fill:#00832d"/><path d="M350.7 338.1H161.3L80.6 477.7c7.9 4.7 17 7 26.4 7h298c9.4 0 18.5-2.6 26.4-7z" style="fill:#2684fc"/><path d="M430.5 182.7 356 53.7c-4.7-8.2-11.4-14.7-19.4-19.4L256 173.9l94.7 164.2h161c0-9.1-2.3-18.2-7-26.4z" style="fill:#ffba00"/></svg></a><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Amazon S3 storage" title="Amazon S3 storage"><svg aria-hidden="true" focusable="false" fill="#569A31" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M20.913 13.147l.12-.895c.947.576 1.258.922 1.354 1.071-.16.031-.562.046-1.474-.176zm-2.174 7.988a.547.547 0 0 0-.005.073c0 .084-.207.405-1.124.768a10.28 10.28 0 0 1-1.438.432c-1.405.325-3.128.504-4.853.504-4.612 0-7.412-1.184-7.412-1.704a.547.547 0 0 0-.005-.073L1.81 5.602c.135.078.28.154.432.227.042.02.086.038.128.057.134.062.272.122.417.18l.179.069c.154.058.314.114.478.168.043.013.084.029.13.043.207.065.423.127.646.187l.176.044c.175.044.353.087.534.127a23.414 23.414 0 0 0 .843.17l.121.023c.252.045.508.085.768.122.071.011.144.02.216.03.2.027.4.053.604.077l.24.027c.245.026.49.05.74.07l.081.009c.275.022.552.04.83.056l.233.012c.21.01.422.018.633.025a33.088 33.088 0 0 0 2.795-.026l.232-.011c.278-.016.555-.034.83-.056l.08-.008c.25-.02.497-.045.742-.072l.238-.026c.205-.024.408-.05.609-.077.07-.01.141-.019.211-.03.261-.037.519-.078.772-.122l.111-.02c.215-.04.427-.082.634-.125l.212-.047c.186-.041.368-.085.546-.13l.166-.042c.225-.06.444-.122.654-.189.04-.012.077-.026.115-.038a10.6 10.6 0 0 0 .493-.173c.058-.021.114-.044.17-.066.15-.06.293-.12.43-.185.038-.017.079-.034.116-.052.153-.073.3-.15.436-.228l-.976 7.245c-2.488-.78-5.805-2.292-7.311-3a1.09 1.09 0 0 0-1.088-1.085c-.6 0-1.088.489-1.088 1.088 0 .6.488 1.089 1.088 1.089.196 0 .378-.056.537-.148 1.72.812 5.144 2.367 7.715 3.15zm-7.42-20.047c5.677 0 9.676 1.759 9.75 2.736l-.014.113c-.01.033-.031.067-.048.101-.015.028-.026.057-.047.087-.024.033-.058.068-.09.102-.028.03-.051.06-.084.09-.038.035-.087.07-.133.105-.04.03-.074.06-.119.091-.053.036-.116.071-.177.107-.05.03-.095.06-.15.09-.068.036-.147.073-.222.11-.059.028-.114.057-.177.085-.084.038-.177.074-.268.111-.068.027-.13.054-.203.082-.097.036-.205.072-.31.107-.075.026-.148.053-.228.079-.111.035-.233.069-.35.103-.085.024-.165.05-.253.073-.124.034-.258.065-.389.098-.093.022-.181.046-.278.068-.139.032-.287.061-.433.091-.098.02-.191.041-.293.06-.155.03-.32.057-.482.084-.1.018-.198.036-.302.052-.166.026-.342.048-.515.072-.11.014-.213.03-.325.044-.181.023-.372.041-.56.06-.11.012-.218.025-.332.036-.188.016-.386.029-.58.043-.122.009-.24.02-.364.028-.207.012-.422.02-.635.028-.12.005-.234.012-.354.016a35.605 35.605 0 0 1-2.069 0c-.12-.004-.234-.011-.352-.016-.214-.008-.43-.016-.637-.028-.122-.008-.238-.02-.36-.027-.195-.015-.394-.028-.584-.044-.11-.01-.215-.024-.324-.035-.19-.02-.384-.038-.568-.06l-.315-.044c-.176-.024-.355-.046-.525-.073-.1-.015-.192-.033-.29-.05-.167-.028-.335-.055-.494-.086-.096-.018-.183-.038-.276-.056-.151-.032-.305-.062-.45-.095-.09-.02-.173-.043-.26-.064-.138-.034-.277-.067-.407-.102-.082-.022-.157-.046-.235-.069a11.75 11.75 0 0 1-.368-.108c-.075-.024-.141-.049-.213-.073-.11-.037-.223-.075-.325-.113-.067-.025-.125-.051-.188-.077-.096-.038-.195-.076-.282-.115-.06-.027-.11-.054-.166-.08-.08-.039-.162-.077-.233-.116-.052-.028-.094-.055-.142-.084-.063-.038-.13-.075-.185-.113-.043-.029-.075-.058-.113-.086-.048-.037-.098-.073-.139-.11-.032-.029-.054-.057-.08-.087-.033-.035-.069-.07-.093-.104-.02-.03-.031-.058-.046-.086-.018-.035-.039-.068-.049-.102l-.015-.113c.076-.977 4.074-2.736 9.748-2.736zm12.182 12.124c-.118-.628-.84-1.291-2.31-2.128l.963-7.16a.531.531 0 0 0 .005-.073C22.16 1.581 16.447 0 11.32 0 6.194 0 .482 1.581.482 3.851a.58.58 0 0 0 .005.072L2.819 21.25c.071 2.002 5.236 2.75 8.5 2.75 1.805 0 3.615-.188 5.098-.531.598-.138 1.133-.3 1.592-.48 1.18-.467 1.789-1.053 1.813-1.739l.945-7.018c.557.131 1.016.197 1.389.197.54 0 .902-.137 1.134-.413aLine truncated
+      <div class="abx-app-row abx-tools-row"><a class="abx-app-name" href="https://abx-dl.archivebox.io/">Tools</a><div class="abx-platforms"><a class="abx-tool" href="https://abx-dl.archivebox.io/"><span>One-shot <code>abx-dl</code> CLI</span></a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/tree/dev/archivebox/mcp">MCP</a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-api-usage">API</a><a class="abx-tool" href="https://web.archivebox.io">Demo</a></div></div>
+    </div></details>
+    __SITE_NAV__
+  </nav>
+</header>

.github/pages/base/marquee.css

@@ -0,0 +1,11 @@
+/* Identical standalone gallery strip across ArchiveBox product sites. */
+/* The root query container measures the usable width, excluding scrollbars. */
+html { container-type: inline-size; }
+.abx-marquee { min-width: 0; width: 100cqw; max-width: none; margin: 24px calc(50% - 50cqw) 40px; padding: 0; overflow: hidden; }
+.abx-marquee .abx-marquee-viewport { width: 100%; overflow-x: auto; scroll-behavior: auto; overscroll-behavior-x: contain; scrollbar-width: thin; padding: 16px 24px 28px; box-sizing: border-box; }
+.abx-marquee .abx-marquee-track { display: flex; align-items: flex-start; gap: 24px; width: max-content; margin-inline: auto; }
+.abx-marquee .abx-marquee-card { display: block; flex: none; width: min-content; overflow: hidden; border-radius: 10px; background: #fff; box-shadow: 0 0 0 1px rgb(0 0 0 / 12%), 0 2px 4px rgb(0 0 0 / 10%), 0 8px 16px rgb(0 0 0 / 16%); text-decoration: none; color: #533344; }
+.abx-marquee .abx-marquee-card img { display: block; height: clamp(240px, 30vw, 400px); width: auto; max-width: none; max-height: none; object-fit: contain; box-shadow: none; border: 0; border-radius: 0; background: transparent; padding: 0; margin: 0; }
+.abx-marquee :focus-visible { outline: 2px solid #9b2854; outline-offset: 3px; }
+.abx-marquee .abx-marquee-viewport:focus-visible { outline-offset: -3px; }
+@media (max-width: 600px) { .abx-marquee .abx-marquee-track { gap: 16px; } .abx-marquee .abx-marquee-card img { height: 240px; } }

.github/pages/base/marquee.js

@@ -0,0 +1,44 @@
+/* Scroll real gallery links once; keep them usable without JavaScript. */
+(() => {
+  document.querySelectorAll('.abx-marquee').forEach(gallery => {
+    const viewport = gallery.querySelector('.abx-marquee-viewport');
+    const reducedMotion = window.matchMedia('(prefers-reduced-motion: reduce)');
+    let paused = reducedMotion.matches;
+    let hovering = false;
+    let focused = false;
+    let visible = false;
+    let frame;
+    let previous;
+    let offset = viewport.scrollLeft;
+    function tick(time) {
+      const distance = viewport.scrollWidth - viewport.clientWidth;
+      if (distance <= 0) return;
+      if (previous !== undefined) {
+        offset = Math.min(offset + Math.min(time - previous, 50) * 0.035, distance);
+        viewport.scrollTo({ left: offset, behavior: 'instant' });
+      }
+      if (offset >= distance - 1) { paused = true; return; }
+      previous = time;
+      frame = window.requestAnimationFrame(tick);
+    }
+    function sync() {
+      window.cancelAnimationFrame(frame);
+      previous = undefined;
+      offset = viewport.scrollLeft;
+      if (!paused && !hovering && !focused && visible && !document.hidden) frame = window.requestAnimationFrame(tick);
+    }
+    viewport.addEventListener('pointerenter', event => { if (event.pointerType === 'mouse') { hovering = true; sync(); } });
+    viewport.addEventListener('pointerleave', () => { hovering = false; sync(); });
+    viewport.addEventListener('focusin', () => { focused = true; sync(); });
+    viewport.addEventListener('focusout', event => { focused = viewport.contains(event.relatedTarget); sync(); });
+    function pauseForInteraction() { paused = true; sync(); }
+    viewport.addEventListener('pointerdown', pauseForInteraction);
+    viewport.addEventListener('wheel', pauseForInteraction, { passive: true });
+    viewport.addEventListener('keydown', pauseForInteraction);
+    reducedMotion.addEventListener('change', () => { paused = reducedMotion.matches; sync(); });
+    document.addEventListener('visibilitychange', sync);
+    new window.IntersectionObserver(([entry]) => { visible = entry.isIntersecting; sync(); }).observe(gallery);
+    new window.ResizeObserver(sync).observe(viewport);
+    viewport.querySelectorAll('img').forEach(image => image.addEventListener('load', sync));
+  });
+})();

.github/pages/marquee.py

@@ -0,0 +1,96 @@
+"""Render homepage screenshot strips from the site's existing gallery HTML."""
+
+import re
+from html import escape
+from html.parser import HTMLParser
+from urllib.parse import urljoin
+
+
+class Gallery(HTMLParser):
+    """Only gallery figures are captures; navigation and decorative images are not."""
+
+    def __init__(self, base):
+        super().__init__()
+        self.base = base
+        self.stack = []
+        self.captures = []
+
+    def handle_starttag(self, tag, attrs):
+        attrs = dict(attrs)
+        if tag == "img" and any(name == "figure" for name, _ in self.stack):
+            context = [item for _, item in self.stack]
+            profile = next(
+                (
+                    item.get("data-profile") or item.get("data-viewport")
+                    for item in reversed(context)
+                    if item.get("data-profile") or item.get("data-viewport")
+                ),
+                None,
+            )
+            if profile not in (None, "desktop"):
+                return
+            source = attrs.get("src", "")
+            if not source or not attrs.get("alt"):
+                return
+            anchor = next((item["id"] for name, item in reversed(self.stack) if name in ("article", "figure") and item.get("id")), "")
+            server = any(item.get("data-platform") == "server" for item in context)
+            self.captures.append(
+                {
+                    "src": urljoin(self.base, source),
+                    "href": self.base + "#" + anchor if anchor else urljoin(self.base, source),
+                    "title": attrs["alt"],
+                    "width": attrs.get("width", ""),
+                    "height": attrs.get("height", ""),
+                    "product": "server" if server else "client",
+                },
+            )
+        if tag not in {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}:
+            self.stack.append((tag, attrs))
+
+    def handle_endtag(self, tag):
+        for index in range(len(self.stack) - 1, -1, -1):
+            if self.stack[index][0] == tag:
+                del self.stack[index:]
+                break
+
+
+def render(output, baseurl="", featured=()):
+    homepage = output / "index.html"
+    text = homepage.read_text()
+    slots = re.findall(r"<!-- ARCHIVEBOX:(MARQUEE(?:-CLIENT|-SERVER)?) -->", text)
+    if not slots:
+        return False
+    base = "/" + baseurl.strip("/") + "/" if baseurl.strip("/") else "/"
+    gallery = Gallery(base + "screenshots/")
+    gallery.feed((output / "screenshots/index.html").read_text())
+    for slot in slots:
+        product = slot.removeprefix("MARQUEE-").lower() if "-" in slot else None
+        captures = [capture for capture in gallery.captures if product is None or capture["product"] == product]
+        priority = {title: index for index, title in enumerate(featured)}
+        captures.sort(key=lambda capture: priority.get(capture["title"], len(priority)))
+        cards = []
+        for capture in captures:
+            dimensions = "".join(f' {key}="{escape(capture[key])}"' for key in ("width", "height") if capture[key])
+            cards.append(
+                f'<a class="abx-marquee-card" href="{escape(capture["href"])}"><img src="{escape(capture["src"])}" alt="{escape(capture["title"])}"{dimensions} loading="lazy" decoding="async"></a>',
+            )
+        viewport_id = slot.lower()
+        content = (
+            f'<section class="abx-marquee" aria-label="App screenshots"><div class="abx-marquee-viewport" id="{viewport_id}" tabindex="0" aria-label="Screenshots. Scroll sideways to explore."><div class="abx-marquee-track">{"".join(cards)}</div></div></section>'
+            if cards
+            else ""
+        )
+        start, end = f"<!-- ARCHIVEBOX:{slot} -->", f"<!-- /ARCHIVEBOX:{slot} -->"
+        replacement = start + "\n" + content + "\n" + end
+        if end in text:
+            text = re.sub(
+                re.escape(start) + r".*?" + re.escape(end),
+                lambda _, replacement=replacement: replacement,
+                text,
+                count=1,
+                flags=re.DOTALL,
+            )
+        else:
+            text = text.replace(start, replacement, 1)
+    homepage.write_text(text)
+    return True

.github/pages/nav.html

@@ -0,0 +1,4 @@
+<!-- Repository-specific navigation and primary CTA. -->
+<a href="__SITE_BASE__#use-cases">Use cases</a><a href="__SITE_BASE__#configuration">Configuration</a><a href="__SITE_BASE__#layout-security">Layout &amp; security</a><a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
+    <a href="https://github.com/ArchiveBox/ArchiveBox">GitHub ↗</a>
+    <a class="abx-cta" href="__SITE_BASE__#install">Get started</a>

.github/pages/site.json

@@ -0,0 +1,16 @@
+{
+  "name": "Web Archive",
+  "url": "https://archivebox.io/",
+  "pages": [
+    "index.html",
+    "screenshots/index.html"
+  ],
+  "marquee_featured": [
+    "Snapshots grid — desktop",
+    "Snapshot View (header collapsed) — desktop",
+    "AI agent — desktop",
+    "Archive results — desktop",
+    "Add URLs — desktop",
+    "Snapshots table — desktop"
+  ]
+}

.github/pages/site.py

@@ -0,0 +1,215 @@
+"""Assemble and verify this repository's static Pages site (standard library only)."""
+
+import argparse
+import json
+import re
+import shutil
+import subprocess
+from datetime import UTC, datetime
+from html import escape, unescape
+from html.parser import HTMLParser
+from pathlib import Path
+from urllib.parse import unquote, urlsplit
+
+from marquee import render as render_marquee
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parents[1]
+
+
+def replace_block(text, name, content):
+    start, end = f"<!-- ARCHIVEBOX:{name} -->", f"<!-- /ARCHIVEBOX:{name} -->"
+    pattern = re.escape(start) + r".*?" + re.escape(end)
+    if re.search(pattern, text, re.DOTALL):
+        return re.sub(
+            pattern,
+            lambda _: f"{start}\n{content}\n{end}",
+            text,
+            count=1,
+            flags=re.DOTALL,
+        )
+    if text.count(start) != 1:
+        raise ValueError(f"Expected one {name} placeholder")
+    return text.replace(start, f"{start}\n{content}\n{end}", 1)
+
+
+def render_metadata(text):
+    """Keep product copy in native templates; render one consistent SEO block."""
+    page = Page()
+    page.feed(text)
+    title = unescape(re.search(r"<title>(.*?)</title>", text, re.DOTALL)[1])
+    meta = page.metadata
+    description = meta.get("description", meta.get("og:description", ""))
+    image = meta.get("og:image", meta.get("twitter:image", ""))
+    if not description or not image or not page.canonical:
+        raise ValueError("Each page needs a description, social image and canonical")
+    values = {
+        "description": description,
+        "robots": "index,follow,max-image-preview:large",
+        "theme-color": "#9b2854",
+        "og:type": "website",
+        "og:site_name": meta.get("og:site_name", "ArchiveBox"),
+        "og:locale": meta.get("og:locale", "en_US"),
+        "og:url": page.canonical,
+        "og:title": title,
+        "og:description": description,
+        "og:image": image,
+        "og:image:type": meta.get("og:image:type", "image/png"),
+        "og:image:width": meta.get("og:image:width", "1200"),
+        "og:image:height": meta.get("og:image:height", "630"),
+        "og:image:alt": meta.get("og:image:alt", title),
+        "twitter:card": "summary_large_image",
+        "twitter:site": "@ArchiveBoxApp",
+        "twitter:title": title,
+        "twitter:description": description,
+        "twitter:image": image,
+        "twitter:image:alt": meta.get("og:image:alt", title),
+    }
+
+    def strip_managed_tag(match):
+        tag = Page()
+        tag.feed(match[0])
+        if tag.canonicals or any(key in values for key in tag.metadata):
+            return ""
+        # System fonts are shared across sites, without a network font dependency.
+        if "fonts.googleapis.com" in match[0] or "fonts.gstatic.com" in match[0]:
+            return ""
+        return match[0]
+
+    text = re.sub(r"<(?:meta|link)\b[^>]*>", strip_managed_tag, text)
+    text = re.sub(r"<title>.*?</title>", "", text, flags=re.DOTALL)
+    tags = [
+        f"<title>{escape(title)}</title>",
+        f'<link rel="canonical" href="{escape(page.canonical, quote=True)}">',
+    ]
+    for key, value in values.items():
+        attr = "property" if key.startswith("og:") else "name"
+        tags.append(f'<meta {attr}="{key}" content="{escape(value, quote=True)}">')
+    if "<!-- ARCHIVEBOX:METADATA -->" not in text:
+        text = text.replace("</head>", "<!-- ARCHIVEBOX:METADATA -->\n</head>", 1)
+    return replace_block(text, "METADATA", "\n".join(tags))
+
+
+def render(output, source=None, baseurl=""):
+    config = json.loads((HERE / "site.json").read_text())
+    output = output.resolve()
+    if source:
+        source = source.resolve()
+        if output == source or output in source.parents or source in output.parents:
+            raise ValueError("Build output must be separate from the source directory")
+        if output == ROOT or output in ROOT.parents:
+            raise ValueError("Build output must not replace a repository")
+        if output.exists():
+            shutil.rmtree(output)
+        shutil.copytree(source, output)
+    revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
+    assets = output / "site-base"
+    assets.mkdir(parents=True, exist_ok=True)
+    has_marquee = render_marquee(output, baseurl, config.get("marquee_featured", []))
+    common_assets = ["chrome.css", "chrome.js"] + (["marquee.css", "marquee.js"] if has_marquee else [])
+    for name in common_assets:
+        shutil.copyfile(HERE / "base" / name, assets / name)
+    for name in config["pages"]:
+        page = output / name
+        text = render_metadata(page.read_text())
+        base = "../" * (len(Path(name).parts) - 1) or "./"
+        header = (HERE / "base/header.html").read_text().replace("__SITE_NAME__", escape(config["name"]))
+        header = header.replace("__SITE_NAV__", (HERE / "nav.html").read_text().strip())
+        text = replace_block(text, "HEADER", header)
+        for kind in ["FOOTER-START", "FOOTER-END"]:
+            text = replace_block(text, kind, (HERE / "base" / f"{kind.lower()}.html").read_text().strip())
+        text = text.replace("__SITE_BASE__", base).replace("__SITE_REVISION__", revision)
+        resources = f'<link rel="stylesheet" href="{base}site-base/chrome.css?v={revision}">\n<script src="{base}site-base/chrome.js?v={revision}" defer></script>'
+        if has_marquee and name == "index.html":
+            resources += f'\n<link rel="stylesheet" href="{base}site-base/marquee.css?v={revision}">\n<script src="{base}site-base/marquee.js?v={revision}" defer></script>'
+        if "<!-- ARCHIVEBOX:ASSETS -->" not in text:
+            text = text.replace("</head>", "<!-- ARCHIVEBOX:ASSETS -->\n</head>", 1)
+        text = replace_block(text, "ASSETS", resources)
+        page.write_text(text)
+    # Site build provenance is independent of all original screenshot manifests.
+    provenance_path = output / "build.json"
+    provenance = json.loads(provenance_path.read_text()) if provenance_path.exists() else {}
+    provenance.update(
+        revision=revision,
+        generatedAt=datetime.now(UTC).isoformat(),
+        site=config["url"],
+    )
+    provenance_path.write_text(json.dumps(provenance, indent=2) + "\n")
+    (output / ".nojekyll").touch()
+    verify(output, baseurl)
+    print(f"Assembled {len(config['pages'])} pages at {output} from {revision}")
+
+
+class Page(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.ids = set()
+        self.metadata = {}
+        self.canonical = ""
+        self.resources = []
+        self.headers = self.footers = self.canonicals = 0
+
+    def handle_starttag(self, tag, attrs):
+        attrs = dict(attrs)
+        if tag == "meta":
+            for key in ["name", "property"]:
+                if attrs.get(key):
+                    self.metadata[attrs[key]] = attrs.get("content", "")
+        if tag == "link" and attrs.get("rel") == "canonical":
+            self.canonical = attrs["href"]
+        if attrs.get("id"):
+            self.ids.add(attrs["id"])
+        classes = attrs.get("class", "").split()
+        self.headers += "abx-header" in classes
+        self.footers += "abx-footer" in classes
+        self.canonicals += tag == "link" and attrs.get("rel") == "canonical"
+        if tag in ["img", "script"] and attrs.get("src"):
+            self.resources.append(attrs["src"])
+        if tag == "link" and attrs.get("rel") in [
+            "stylesheet",
+            "icon",
+            "apple-touch-icon",
+        ]:
+            self.resources.append(attrs["href"])
+
+
+def verify(output, baseurl=""):
+    config = json.loads((HERE / "site.json").read_text())
+    output = output.resolve()
+    prefix = "/" + baseurl.strip("/") if baseurl.strip("/") else ""
+    for name in config["pages"]:
+        path = output / name
+        text = path.read_text()
+        page = Page()
+        page.feed(text)
+        if (page.headers, page.footers, page.canonicals) != (1, 1, 1):
+            raise ValueError(f"{name}: expected exactly one shared header, footer and canonical")
+        if "__SITE_" in text:
+            raise ValueError(f"{name}: unresolved site placeholder")
+        for resource in page.resources:
+            url = urlsplit(resource)
+            if url.scheme or url.netloc or not url.path:
+                continue
+            resource_path = unquote(url.path)
+            if prefix and resource_path.startswith(prefix + "/"):
+                resource_path = resource_path[len(prefix) :]
+            local = (output / resource_path.lstrip("/") if resource_path.startswith("/") else path.parent / resource_path).resolve()
+            if not local.is_relative_to(output) or not local.is_file():
+                raise ValueError(f"{name}: missing local resource {resource}")
+        for asset in ["chrome.css", "chrome.js"]:
+            if (output / "site-base" / asset).read_bytes() != (HERE / "base" / asset).read_bytes():
+                raise ValueError(f"{name}: stale common asset {asset}")
+    print(f"Verified shared layout and local resources for {len(config['pages'])} pages")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("command", choices=["render", "verify"])
+    parser.add_argument("output", type=Path)
+    parser.add_argument("--source", type=Path)
+    parser.add_argument("--baseurl", default="")
+    args = parser.parse_args()
+    if args.command == "render":
+        render(args.output, args.source, args.baseurl)
+    else:
+        verify(args.output, args.baseurl)

.github/pages/verify.py

@@ -0,0 +1,255 @@
+"""Check the built site in real Chromium at desktop and phone widths."""
+
+import argparse
+import json
+import re
+from functools import partial
+from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+from html.parser import HTMLParser
+from pathlib import Path
+from threading import Thread
+from urllib.parse import unquote, urljoin, urlsplit
+
+from playwright.sync_api import expect, sync_playwright
+
+HERE = Path(__file__).resolve().parent
+
+
+class Handler(SimpleHTTPRequestHandler):
+    def handle(self):
+        # Browser navigation can cancel an in-flight lazy image response.
+        try:
+            super().handle()
+        except (BrokenPipeError, ConnectionResetError):
+            pass
+
+    def log_message(self, *args):
+        pass
+
+
+class ScreenshotReferences(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.references = []
+        self.ids = set()
+
+    def handle_starttag(self, tag, attributes):
+        attrs = dict(attributes)
+        if attrs.get("id"):
+            self.ids.add(attrs["id"])
+        for attribute in ("src", "href"):
+            if attrs.get(attribute):
+                self.references.append(attrs[attribute])
+
+
+def verify_screenshot_references(output, canonical):
+    """Check gallery embeds on every static page, including optional translations."""
+    pages = {}
+    for path in output.rglob("*.html"):
+        page = ScreenshotReferences()
+        page.feed(path.read_text())
+        pages[path.resolve()] = page
+    for path, page in pages.items():
+        page_url = urljoin(canonical, path.relative_to(output.resolve()).as_posix())
+        for reference in page.references:
+            url = urlsplit(urljoin(page_url, reference))
+            if url.netloc != urlsplit(canonical).netloc or "/screenshots/" not in url.path:
+                continue
+            target = output / unquote(url.path).lstrip("/")
+            if target.is_dir():
+                target /= "index.html"
+            target = target.resolve()
+            assert target.is_relative_to(output.resolve()) and target.is_file(), f"{path}: missing screenshot target {reference}"
+            assert not re.match(r"^\d+-", target.name), f"{path}: screenshot filename depends on capture order: {reference}"
+            if url.fragment and target in pages:
+                assert unquote(url.fragment) in pages[target].ids, f"{path}: missing screenshot anchor {reference}"
+
+
+def verify(output, evidence):
+    config = json.loads((HERE / "site.json").read_text())
+    verify_screenshot_references(output, config["url"])
+    evidence.mkdir(parents=True, exist_ok=True)
+    server = ThreadingHTTPServer(("127.0.0.1", 0), partial(Handler, directory=str(output.resolve())))
+    thread = Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    origin = f"http://127.0.0.1:{server.server_port}"
+    try:
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(ignore_default_args=["--hide-scrollbars"])
+            for name in config["pages"]:
+                route = name.removesuffix("index.html")
+                page = browser.new_page(reduced_motion="reduce")
+                errors = []
+                missing = []
+                page.on("pageerror", lambda error, errors=errors: errors.append(str(error)))
+                page.on(
+                    "response",
+                    lambda response, missing=missing: (
+                        missing.append(response.url) if response.url.startswith(origin) and response.status >= 400 else None
+                    ),
+                )
+                for width in [1716, 2560, 390]:
+                    page.set_viewport_size({"width": width, "height": 900})
+                    page.goto(f"{origin}/{route}", wait_until="domcontentloaded")
+                    header = page.locator(".abx-header")
+                    expect(header).to_have_count(1)
+                    expect(header).to_be_visible()
+                    expect(header).to_be_in_viewport()
+                    brand = header.locator(".abx-brand")
+                    expect(brand).to_be_visible()
+                    expect(brand).to_contain_text("ArchiveBox")
+                    expect(brand.locator(".abx-logo")).to_be_visible()
+                    expect(brand.locator(".abx-logo")).to_be_in_viewport()
+                    expect(header).to_have_css("display", "flex")
+                    expect(page.locator(".abx-footer")).to_have_count(1)
+                    expect(page.locator(".abx-footer-column")).to_have_count(3)
+                    expect(header.locator(".abx-cta")).to_be_visible()
+                    menu = header.locator(".abx-apps")
+                    menu.locator("summary").focus()
+                    page.keyboard.press("Enter")
+                    expect(menu).to_have_attribute("open", "")
+                    expect(menu.locator("a").first).to_be_visible()
+                    page.keyboard.press("Escape")
+                    expect(menu).not_to_have_attribute("open", "")
+                    menu.locator("summary").click()
+                    box = header.bounding_box()
+                    assert box
+                    page.mouse.click(box["x"] + 2, box["y"] + 5)
+                    expect(menu).not_to_have_attribute("open", "")
+                    assert page.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth"), (
+                        f"{name}: overflow at {width}"
+                    )
+                    if not route and "screenshots/index.html" in config["pages"]:
+                        strips = page.locator(".abx-marquee")
+                        assert strips.count() > 0, "Homepage must use its screenshot gallery"
+                        for strip in strips.all():
+                            strip.scroll_into_view_if_needed()
+                            bounds = strip.bounding_box()
+                            usable_width = page.evaluate("document.documentElement.clientWidth")
+                            assert bounds and abs(bounds["x"]) < 1
+                            assert abs(bounds["width"] - usable_width) < 1
+                            assert not strip.inner_text().strip(), "Screenshot strips should have no visible captions or links"
+                            cards = strip.locator(".abx-marquee-card")
+                            assert cards.count() > 0, "Screenshot strip is empty"
+                            expect(strip.locator("button")).to_have_count(0)
+                            expect(strip.locator("a:not(.abx-marquee-card)")).to_have_count(0)
+                            viewport = strip.locator(".abx-marquee-viewport")
+                            overflowing = viewport.evaluate(
+                                "node => node.scrollWidth > node.clientWidth",
+                            )
+                            if overflowing:
+                                page.mouse.move(0, 0)
+                                page.emulate_media(reduced_motion="no-preference")
+                                initial = viewport.evaluate("node => node.scrollLeft")
+                                page.wait_for_function(
+                                    "([node, initial]) => node.scrollLeft > initial",
+                                    arg=[viewport.element_handle(), initial],
+                                )
+                                viewport.hover()
+                                stopped = viewport.evaluate("node => node.scrollLeft")
+                                viewport.evaluate(
+                                    "node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
+                                )
+                                assert viewport.evaluate("node => node.scrollLeft") == stopped
+                                page.emulate_media(reduced_motion="reduce")
+                                page.mouse.move(0, 0)
+                                stopped = viewport.evaluate("node => node.scrollLeft")
+                                viewport.evaluate(
+                                    "node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
+                                )
+                                assert viewport.evaluate("node => node.scrollLeft") == stopped
+                                viewport.focus()
+                                page.keyboard.press("ArrowRight")
+                                assert viewport.evaluate("node => node.scrollWidth > node.clientWidth")
+                            else:
+                                centered = viewport.evaluate(
+                                    """node => {
+                                        const view = node.getBoundingClientRect();
+                                        const track = node.querySelector('.abx-marquee-track').getBoundingClientRect();
+                                        return Math.abs((track.left - view.left) - (view.right - track.right)) < 2;
+                                    }""",
+                                )
+                                assert centered, "Short screenshot strips must be centered"
+                            for card in cards.all():
+                                href = card.get_attribute("href")
+                                assert href and "/screenshots/" in href
+                                response = page.request.get(urljoin(page.url, href))
+                                assert response.ok, f"Broken screenshot link: {href}"
+                                anchor = urlsplit(href).fragment
+                                if anchor:
+                                    assert f'id="{anchor}"' in response.text(), f"Missing screenshot anchor: {href}"
+                    # Decode every local content image using the real browser loader,
+                    # including images in offscreen carousel tracks and lazy galleries.
+                    images = page.locator("main img").evaluate_all("nodes => [...new Set(nodes.map(image => image.src))]")
+                    local_images = [url for url in images if url.startswith(origin + "/")]
+                    page.evaluate(
+                        """async urls => {
+                        for (const url of urls) {
+                            const image = new Image();
+                            image.src = url;
+                            await image.decode();
+                            if (!image.naturalWidth) throw new Error(`Broken image: ${url}`);
+                        }
+                    }""",
+                        local_images,
+                    )
+                    page.evaluate("scrollTo({top: 0, behavior: 'instant'})")
+                    page.evaluate(
+                        "() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
+                    )
+                    page.screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}{width}.png"))
+                    page.locator(".abx-footer").screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}footer-{width}.png"))
+                    for link in header.locator(".abx-nav > a").all():
+                        url = urlsplit(link.get_attribute("href") or "")
+                        if url.scheme or url.netloc:
+                            continue
+                        response = page.request.get(urljoin(page.url, url.geturl()))
+                        assert response.ok, f"Broken navigation: {url.geturl()}"
+                        if url.fragment:
+                            body = response.text()
+                            assert f'id="{url.fragment}"' in body or f"id='{url.fragment}'" in body, (
+                                f"Missing navigation anchor: {url.geturl()}"
+                            )
+                    if route == "" and page.locator("#resources").count():
+                        page.goto(f"{origin}/#resources")
+                        expect(page.locator("#resources")).to_be_visible()
+                assert not errors, errors
+                assert not missing, missing
+                page.close()
+                context = browser.new_context(java_script_enabled=False, viewport={"width": 390, "height": 900})
+                plain = context.new_page()
+                plain.goto(f"{origin}/{route}", wait_until="domcontentloaded")
+                expect(plain.locator(".abx-header")).to_be_visible()
+                expect(plain.locator(".abx-header .abx-brand")).to_be_in_viewport()
+                expect(plain.locator(".abx-header .abx-logo")).to_be_visible()
+                plain.locator(".abx-apps summary").click()
+                expect(plain.locator(".abx-app-links a").first).to_be_visible()
+                expect(plain.locator(".abx-footer-column a").first).to_be_visible()
+                if not route and "screenshots/index.html" in config["pages"]:
+                    assert plain.locator(".abx-marquee-card").count() > 0
+                    for strip in plain.locator(".abx-marquee").all():
+                        bounds = strip.bounding_box()
+                        usable_width = plain.evaluate("document.documentElement.clientWidth")
+                        assert bounds and abs(bounds["x"]) < 1
+                        assert abs(bounds["width"] - usable_width) < 1
+                    assert plain.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth")
+                    expect(plain.locator(".abx-marquee button")).to_have_count(0)
+                    expect(plain.locator(".abx-marquee-card").first).to_be_visible()
+                context.close()
+                print(
+                    f"PASS {name}: desktop/mobile, keyboard, no-JS, images and resources",
+                    flush=True,
+                )
+            browser.close()
+    finally:
+        server.shutdown()
+        server.server_close()
+        thread.join()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("output", type=Path)
+    parser.add_argument("--evidence", type=Path, default=Path("test-results/site"))
+    args = parser.parse_args()
+    verify(args.output, args.evidence)

.github/workflows/ci.yml

@@ -52,7 +52,7 @@ jobs:
     name: Tested Docker images
     uses: ./.github/workflows/docker.yml
     with:
-      push_digests: false
+      push_digests: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
     secrets: inherit
 
   required:

.github/workflows/deploy-publicsite.yml

@@ -1,79 +1,87 @@
-name: Deploy Publicsite to GitHub Pages
+name: Publish website
 
 on:
   push:
-    branches:
-      - dev
-    paths:
-      - publicsite/**
-      - archivebox/**
-      - bin/collect_ui_screenshots.*
-      - bin/generate_ui_screenshot_gallery.py
-      - bin/take_screenshot.js
-      - docs/Screenshots.md
-      - pyproject.toml
-      - uv.lock
-      - .github/workflows/deploy-publicsite.yml
+    branches: [dev]
+  pull_request:
   workflow_dispatch:
+  workflow_run:
+    workflows: ['Capture publicsite screenshots']
+    types: [completed]
+    branches: [dev]
 
 permissions:
   contents: read
-  pages: write
-  id-token: write
+  actions: read
 
+env:
+  # Repository-specific output; all publication logic below is a local copy.
+  SITE_OUTPUT: _site
+  DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+
+# One production queue covers restore, build and deploy. PRs have separate queues.
 concurrency:
-  group: ${{ github.event_name == 'workflow_dispatch' && format('github-pages-publicsite-manual-{0}', github.run_id) || 'github-pages-publicsite' }}
-  cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
+  group: pages-${{ github.event_name == 'pull_request' && github.ref || 'production' }}
+  cancel-in-progress: false
 
 jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    outputs:
+      revision: ${{ steps.source.outputs.revision }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.ref || env.DEFAULT_BRANCH }}
+          persist-credentials: false
+      - name: Record website source revision
+        id: source
+        run: echo "revision=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      # BEGIN REPOSITORY-SPECIFIC BUILD
+      - uses: astral-sh/setup-uv@v7
+      - name: Restore latest complete screenshots
+        run: uv run --no-project python bin/restore_site_screenshots.py publicsite/screenshots
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - name: Render current gallery layout without recapturing images
+        run: uv run --no-project python bin/generate_ui_screenshot_gallery.py refresh publicsite/screenshots/index.html
+      - name: Assemble website with the local base components
+        run: uv run --no-project python .github/pages/site.py render _site --source publicsite
+      # END REPOSITORY-SPECIFIC BUILD
+
+      - name: Verify website in Chromium
+        run: |
+          uv run --no-config --no-project --with playwright==1.63.0 playwright install --with-deps chromium
+          uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py "$SITE_OUTPUT" --evidence "$RUNNER_TEMP/site-evidence"
+      - name: Save website verification evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: site-verification
+          path: ${{ runner.temp }}/site-evidence
+          if-no-files-found: warn
+          retention-days: 14
+
+      - name: Upload website
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ${{ env.SITE_OUTPUT }}
+
   deploy:
+    if: github.event_name != 'pull_request'
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-24.04
     steps:
-      - name: Checkout
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Setup Python and uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: false
-
-      - name: Install ArchiveBox and screenshot dependencies
-        run: uv sync --no-cache --frozen
-
-      - name: Build UI screenshot gallery
-        run: bin/collect_ui_screenshots.sh
-        env:
-          BASE_URL: http://archivebox.localhost:8000
-
-      - name: Verify gallery was generated from this exact revision
-        run: |
-          uv run --no-cache --no-sync python - <<'PY'
-          import json
-          import os
-          import tomllib
-          from pathlib import Path
-
-          provenance = json.loads(Path("publicsite/screenshots/build.json").read_text())
-          version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
-          assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance
-          assert provenance["version"] == version, provenance
-          assert provenance["capture_count"] >= 3 * 35, provenance
-          assert len(provenance["files"]) == provenance["capture_count"], provenance
-          PY
-
-      - name: Setup Pages
-        uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
-
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
-        with:
-          path: ./publicsite
-
-      - name: Deploy to GitHub Pages
+      - uses: actions/configure-pages@v5
+      - uses: actions/deploy-pages@v4
         id: deployment
-        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4

.github/workflows/docker.yml

@@ -13,6 +13,22 @@ on:
         required: false
         default: true
         type: boolean
+  workflow_dispatch:
+    inputs:
+      source_ref:
+        description: Exact 40-character commit SHA to build
+        required: true
+        type: string
+      push_digests:
+        description: Push validated per-platform images by digest for the release workflow
+        required: false
+        default: true
+        type: boolean
+      full_tests:
+        description: Run storage, CLI, and size validation in addition to release smoke checks
+        required: false
+        default: true
+        type: boolean
 
 env:
   DOCKERHUB_IMAGE: archivebox/archivebox
@@ -42,6 +58,18 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          ref: ${{ inputs.source_ref || github.sha }}
+
+      - name: Verify exact source commit
+        if: github.event_name == 'workflow_dispatch'
+        shell: bash
+        env:
+          SOURCE_REF: ${{ inputs.source_ref }}
+        run: |
+          set -Eeuo pipefail
+          [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$(git rev-parse HEAD)" == "$SOURCE_REF" ]]
 
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
@@ -183,7 +211,7 @@ jobs:
           {
             echo 'labels<<EOF'
             echo "org.opencontainers.image.version=${VERSION}"
-            echo "org.opencontainers.image.revision=${GITHUB_SHA}"
+            echo "org.opencontainers.image.revision=${{ inputs.source_ref || github.sha }}"
             echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
             echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
             echo 'EOF'
@@ -203,7 +231,7 @@ jobs:
           labels: ${{ steps.docker_meta.outputs.labels }}
           build-args: |
             ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
+            ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
           pull: true
           platforms: ${{ matrix.platform }}
           # Keep the released image as the one cache authority. A second GHA
@@ -232,7 +260,7 @@ jobs:
           labels: ${{ steps.docker_meta.outputs.labels }}
           build-args: |
             ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
+            ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
           pull: true
           platforms: ${{ matrix.platform }}
           # Test the same layer lineage users receive. Reusing :dev avoids a
@@ -246,13 +274,34 @@ jobs:
         if: inputs.push_digests
         run: echo ${{ steps.docker_build.outputs.digest }}
 
+      - name: Verify OpenCode install with a read-only browser profile
+        shell: bash
+        env:
+          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
+        run: |
+          set -Eeuo pipefail
+          PROFILE_DIR="$(mktemp -d)"
+          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' \
+            --volume "$PROFILE_DIR:/home/archivebox/.config/chromium:ro" \
+            "$IMAGE" bash -c '
+              set -Eeuo pipefail
+              test -w "$XDG_CONFIG_HOME"
+              test ! -w "$XDG_CONFIG_HOME/chromium"
+              # Check the shipped executable before the installer can repair
+              # an image that accidentally omitted this opt-in dependency.
+              "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
+              archivebox install opencode
+              "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
+            '
+
       - name: Validate exact built image version and commit
         if: ${{ !inputs.push_digests }}
         shell: bash
         run: |
           set -Eeuo pipefail
-          IMAGE="${{ matrix.local_tag }}"
-          SHORT_SHA="${GITHUB_SHA::7}"
+          IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}"
+          SHORT_SHA="${{ (inputs.source_ref || github.sha) }}"
+          SHORT_SHA="${SHORT_SHA::7}"
 
           DATA_DIR="$(mktemp -d)"
           "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
@@ -296,8 +345,8 @@ jobs:
             MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${IMAGE_DIGEST}")"
           fi
           COMPRESSED_SIZE="$(MANIFEST="$MANIFEST" "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); print(manifest["config"]["size"] + sum(layer["size"] for layer in manifest["layers"]))')"
-          LIMIT=$((750 * 1024 * 1024))
-          echo "${DIGEST} compressed_size=$((COMPRESSED_SIZE / 1024 / 1024)) MiB limit=750 MiB"
+          LIMIT=$((1025 * 1024 * 1024))
+          echo "${DIGEST} compressed_size=$((COMPRESSED_SIZE / 1024 / 1024)) MiB limit=1025 MiB"
           (( COMPRESSED_SIZE < LIMIT ))
 
       - name: Resolve real Docker CLI test dependencies through abxpkg
@@ -375,14 +424,20 @@ jobs:
 
           NFS_TEST_DIR="$NFS_MOUNT" \
           SMB_TEST_DIR="$SMB_MOUNT" \
-          IMAGE="${{ matrix.local_tag }}" \
+          IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
             "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
 
+      - name: Validate Docker port compatibility
+        shell: bash
+        env:
+          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
+        run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
+
       - name: Validate Docker CLI add/list and compose behavior
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash
         env:
-          ARCHIVEBOX_IMAGE: ${{ matrix.local_tag }}
+          ARCHIVEBOX_IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
         run: |
           set -Eeuo pipefail
           "$MKDIR_BINARY" "${{ github.workspace }}/data"
@@ -401,13 +456,13 @@ jobs:
           "$DOCKER_BINARY" compose run --rm archivebox init
           "$DOCKER_BINARY" compose up -d --wait
           "$DOCKER_BINARY" network connect archivebox_default www-nginx
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/public/')"
-          admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js')"
+          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
+          admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
           [[ "$index_html" == *ArchiveBox* ]]
           [[ "$admin_js" == *window.django* ]]
 
           "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/public/')"
+          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
           [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
           "$DOCKER_BINARY" compose down
 
@@ -423,7 +478,7 @@ jobs:
             '${{ runner.temp }}/archivebox-image.tar' \
             '${{ steps.docker_build_pr.outputs.digest }}' \
             "$LOCAL_IMAGE_ID" \
-            "$((750 * 1024 * 1024))" <<'PY'
+            "$((1025 * 1024 * 1024))" <<'PY'
           from hashlib import sha256
           import json
           import os
@@ -503,6 +558,7 @@ jobs:
           mkdir -p /tmp/digests
           digest="${{ steps.docker_build.outputs.digest }}"
           touch "/tmp/digests/${digest#sha256:}"
+          printf '%s\n' '${{ inputs.source_ref || github.sha }}' > /tmp/digests/source-ref
 
       - name: Upload digest
         if: inputs.push_digests

.github/workflows/docs.yml

@@ -146,7 +146,9 @@ jobs:
           fi
       - name: Build the exact checkout's documentation image
         if: ${{ matrix.environment == 'docker' }}
-        run: '"$DOCKER_BINARY" build --tag archivebox/archivebox:dev .'
+        run: |
+          ABX_DL_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(dep.removeprefix("abx-dl==") for dep in tomllib.load(open("pyproject.toml", "rb"))["project"]["dependencies"] if dep.startswith("abx-dl==")))')"
+          "$DOCKER_BINARY" build --build-arg "ABX_DL_IMAGE=archivebox/abx-dl:${ABX_DL_VERSION}" --tag archivebox/archivebox:dev .
       - name: Run the Ubuntu, Docker, or macOS documentation lane
         if: ${{ !contains(fromJSON('["root", "freebsd", "openbsd"]'), matrix.environment) }}
         run: uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'

.github/workflows/release-candidate.yml

@@ -2,7 +2,7 @@ name: Release candidate
 
 on:
   push:
-    branches: [dev]
+    branches: [dev, main]
 
 permissions:
   actions: read
@@ -10,7 +10,7 @@ permissions:
   packages: write
 
 concurrency:
-  group: release-candidate-dev
+  group: release-candidate-${{ github.ref_name }}
   cancel-in-progress: true
 
 jobs:
@@ -25,17 +25,31 @@ jobs:
           submodules: true
           token: ${{ secrets.RELEASE_GH_TOKEN }}
 
+      - name: Keep synchronized stable branches on the released version
+        id: channel
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
+          MAIN_SHA="$(git ls-remote origin refs/heads/main | cut -f1)"
+          if [[ "$GITHUB_REF_NAME" == dev && "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ && "$MAIN_SHA" == "$GITHUB_SHA" ]]; then
+            echo 'stable=true' >> "$GITHUB_OUTPUT"
+          else
+            echo 'stable=false' >> "$GITHUB_OUTPUT"
+          fi
+
       - id: version
+        if: steps.channel.outputs.stable != 'true'
         uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
         with:
           package: archivebox
-          branch: dev
+          branch: ${{ github.ref_name }}
           tag-prefix: v
-          version-scheme: rc
+          version-scheme: ${{ github.ref_name == 'main' && 'patch' || 'rc' }}
 
   python-artifacts:
     needs: prepare
-    if: needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved'
+    if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
     uses: ./.github/workflows/pip.yml
     with:
       full_tests: false
@@ -43,9 +57,23 @@ jobs:
 
   docker-digests:
     needs: prepare
-    if: needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved'
+    if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
     uses: ./.github/workflows/docker.yml
     with:
       push_digests: true
       full_tests: false
     secrets: inherit
+
+  opencode:
+    needs: prepare
+    if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
+    strategy:
+      fail-fast: false
+      matrix:
+        test_file:
+          - archivebox/tests/test_opencode_agent.py
+          - archivebox/tests/test_opencode_browser.py
+    uses: ./.github/workflows/test-parallel.yml
+    with:
+      test_file: ${{ matrix.test_file }}
+    secrets: inherit

.github/workflows/release.yml

@@ -5,30 +5,53 @@ on:
     workflows: [CI, Release candidate]
     types: [completed]
     branches: [dev, main]
+  workflow_dispatch:
+    inputs:
+      release_branch:
+        description: Branch being promoted
+        required: true
+        type: choice
+        options: [main, dev]
+      release_sha:
+        description: Exact tested commit to release
+        required: true
+        type: string
+      ci_run_id:
+        description: Successful CI run containing the tested Python artifacts
+        required: true
+        type: string
+      docker_digest_run_id:
+        description: Manual Docker digest build run containing the tested image digests
+        required: true
+        type: string
 
 permissions: {}
 
 concurrency:
-  group: release-${{ github.repository }}-${{ github.event.workflow_run.head_branch }}
+  group: release-${{ github.repository }}-${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
   cancel-in-progress: false
 
 env:
   DOCKERHUB_IMAGE: archivebox/archivebox
   GHCR_IMAGE: ghcr.io/archivebox/archivebox
-  RELEASE_BRANCH: ${{ github.event.workflow_run.head_branch }}
-  RELEASE_SHA: ${{ github.event.workflow_run.head_sha }}
-  CI_RUN_ID: ${{ github.event.workflow_run.id }}
+  RELEASE_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
+  RELEASE_SHA: ${{ github.event_name == 'workflow_dispatch' && inputs.release_sha || github.event.workflow_run.head_sha }}
 
 jobs:
   candidate:
     if: >-
       github.event.workflow_run.conclusion == 'success' &&
       github.event.workflow_run.event == 'push' &&
-      ((github.event.workflow_run.head_branch == 'dev' && github.event.workflow_run.name == 'Release candidate') ||
-       (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.name == 'CI'))
+      (github.event.workflow_run.head_branch == 'dev' || github.event.workflow_run.head_branch == 'main') ||
+      (github.event_name == 'workflow_dispatch' && inputs.release_branch == 'main')
     runs-on: ubuntu-24.04
+    permissions:
+      actions: read
+      contents: read
     outputs:
-      ready: ${{ steps.owner.outputs.ready }}
+      ready: ${{ steps.verified.outputs.ready }}
+      artifact_run_id: ${{ steps.verified.outputs.artifact_run_id }}
+      digest_run_id: ${{ steps.verified.outputs.digest_run_id }}
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
@@ -37,19 +60,81 @@ jobs:
       - id: owner
         shell: bash
         run: |
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            echo 'ready=true' >> "$GITHUB_OUTPUT"
+          set -Eeuo pipefail
+          BRANCH_SHA="$(git ls-remote origin "refs/heads/$RELEASE_BRANCH" | cut -f1)"
+          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && "$BRANCH_SHA" != "$RELEASE_SHA" ]]; then
+            echo 'ready=false' >> "$GITHUB_OUTPUT"
             exit 0
           fi
           VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
+          if [[ "$RELEASE_BRANCH" == main ]]; then
+            TAG_TARGET="$(git ls-remote origin "refs/tags/v$VERSION" | cut -f1)"
+            if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && -n "$TAG_TARGET" && "$TAG_TARGET" != "$RELEASE_SHA" ]]; then
+              echo 'ready=false' >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo 'ready=true' >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          # A stable release fast-forwards dev to main. Only main publishes it;
+          # the resulting dev workflow is not a new release candidate.
+          if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]]; then
+            echo 'ready=false' >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
           TARGET="$(git ls-remote origin "refs/tags/release-candidate/$VERSION" | awk '{print $1}')"
           [[ "$TARGET" == "$RELEASE_SHA" ]] && echo 'ready=true' >> "$GITHUB_OUTPUT" || echo 'ready=false' >> "$GITHUB_OUTPUT"
 
+      - name: Require successful full CI for this commit
+        id: verified
+        if: steps.owner.outputs.ready == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          MANUAL_CI_RUN_ID: ${{ inputs.ci_run_id }}
+          MANUAL_DIGEST_RUN_ID: ${{ inputs.docker_digest_run_id }}
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          if [[ "$GITHUB_EVENT_NAME" == workflow_dispatch ]]; then
+            CI_RUN_ID="$MANUAL_CI_RUN_ID"
+            DIGEST_RUN_ID="$MANUAL_DIGEST_RUN_ID"
+            CI_STATUS="$(gh run view "$CI_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status,conclusion,headSha,event,name \
+              --jq '[.status, .conclusion, .headSha, .event, .name] | @tsv')"
+            [[ "$CI_STATUS" == "$(printf 'completed\tsuccess\t%s\tpush\tCI' "$RELEASE_SHA")" ]]
+          else
+            CANDIDATE_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow release-candidate.yml \
+              --commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
+              --json databaseId,status,conclusion \
+              --jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
+            CI_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml \
+              --commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
+              --json databaseId,status,conclusion \
+              --jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
+            [[ -n "$CANDIDATE_RUN_ID" && -n "$CI_RUN_ID" ]] || exit 0
+            if [[ "$RELEASE_BRANCH" == dev ]]; then
+              DIGEST_RUN_ID="$CANDIDATE_RUN_ID"
+            else
+              DIGEST_RUN_ID="$CI_RUN_ID"
+            fi
+          fi
+          if [[ "$RELEASE_BRANCH" == dev && "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
+            ARTIFACT_RUN_ID="$CANDIDATE_RUN_ID"
+          else
+            ARTIFACT_RUN_ID="$CI_RUN_ID"
+          fi
+          {
+            echo 'ready=true'
+            echo "artifact_run_id=$ARTIFACT_RUN_ID"
+            echo "digest_run_id=$DIGEST_RUN_ID"
+          } >> "$GITHUB_OUTPUT"
+
   python-release:
     name: Publish tested Python artifacts
     needs: candidate
     if: needs.candidate.outputs.ready == 'true'
     runs-on: ubuntu-24.04
+    env:
+      CI_RUN_ID: ${{ needs.candidate.outputs.artifact_run_id }}
     environment: pypi
     permissions:
       actions: read
@@ -162,20 +247,37 @@ jobs:
   docker-release:
     name: Publish tested multiarch image
     if: >-
-      github.event.workflow_run.conclusion == 'success' &&
-      github.event.workflow_run.event == 'push'
-    needs: python-release
+      (github.event_name == 'workflow_dispatch') ||
+      (github.event.workflow_run.conclusion == 'success' &&
+       github.event.workflow_run.event == 'push')
+    needs: [candidate, python-release]
     runs-on: ubuntu-24.04
+    env:
+      DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
     permissions:
       actions: read
-      contents: read
+      contents: write
       packages: write
     steps:
       - name: Checkout the tested commit
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
-          fetch-depth: 1
+          fetch-depth: 0
           ref: ${{ env.RELEASE_SHA }}
+          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
+
+      - name: Fast-forward dev to the tested stable release
+        if: env.RELEASE_BRANCH == 'main' && needs.python-release.outputs.release_ready == 'true'
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          git fetch --no-tags origin main dev
+          MAIN_SHA="$(git rev-parse origin/main)"
+          DEV_SHA="$(git rev-parse origin/dev)"
+          if [[ "$MAIN_SHA" == "$RELEASE_SHA" && "$DEV_SHA" != "$RELEASE_SHA" ]] &&
+             git merge-base --is-ancestor "$DEV_SHA" "$RELEASE_SHA"; then
+            git push origin "$RELEASE_SHA:refs/heads/dev"
+          fi
 
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
@@ -198,13 +300,14 @@ jobs:
             --lib="$ABXPKG_LIB_DIR" \
             --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
             --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null
-          for name in uv git jq docker; do
+          for name in uv gh git jq docker; do
             test -L "$ABXPKG_LIB_DIR/env/bin/$name"
             test -x "$ABXPKG_LIB_DIR/env/bin/$name"
           done
           {
             echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
             echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
+            echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
             echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
             echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
           } >> "$GITHUB_ENV"
@@ -232,16 +335,31 @@ jobs:
           merge-multiple: true
           github-token: ${{ github.token }}
           repository: ${{ github.repository }}
-          run-id: ${{ env.CI_RUN_ID }}
+          run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
+
+      - name: Verify exact successful digest build run
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -Eeuo pipefail
+          RUN_STATUS="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status --jq '.status')"
+          RUN_CONCLUSION="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json conclusion --jq '.conclusion')"
+          [[ "$RUN_STATUS" == completed && "$RUN_CONCLUSION" == success ]]
+          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
+            RUN_SHA="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json headSha --jq '.headSha')"
+            [[ "$RUN_SHA" == "$RELEASE_SHA" ]]
+          fi
 
       - name: Verify digest set and collect tags
         id: docker_meta
         shell: bash
         run: |
           set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
           [[ "${#DIGESTS[@]}" -eq 2 ]]
           for digest in "${DIGESTS[@]}"; do [[ "$digest" =~ ^[0-9a-f]{64}$ ]]; done
+          [[ "$(< /tmp/digests/source-ref)" == "$RELEASE_SHA" ]]
 
           VERSION='${{ needs.python-release.outputs.version }}'
           SHORT_SHA="${RELEASE_SHA::12}"
@@ -251,12 +369,25 @@ jobs:
             TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")"
             TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
           fi
+          SYNC_DEV=false
+          if [[ "$RELEASE_BRANCH" == main ]]; then
+            $GIT_BINARY fetch --no-tags origin main dev
+            MAIN_TARGET="$($GIT_BINARY rev-parse origin/main)"
+            DEV_TARGET="$($GIT_BINARY rev-parse origin/dev)"
+            # Recovery can add CI/docs fixes after the immutable release tag.
+            # Both branches and every shipped application file must agree.
+            if [[ "$MAIN_TARGET" == "$DEV_TARGET" ]] &&
+               $GIT_BINARY diff --quiet "$RELEASE_SHA" "$MAIN_TARGET" -- . ':!.github' ':!README.md'; then
+              SYNC_DEV=true
+            fi
+          fi
           {
             echo 'dockerhub_tags<<EOF'
             if [[ "$RELEASE_BRANCH" == dev ]]; then
               echo "${DOCKERHUB_IMAGE}:dev"
             elif [[ "$RELEASE_BRANCH" == main ]]; then
               echo "${DOCKERHUB_IMAGE}:latest"
+              [[ "$SYNC_DEV" == true ]] && echo "${DOCKERHUB_IMAGE}:dev"
             fi
             echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
             [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${DOCKERHUB_IMAGE}:${VERSION}"
@@ -266,6 +397,7 @@ jobs:
               echo "${GHCR_IMAGE}:dev"
             elif [[ "$RELEASE_BRANCH" == main ]]; then
               echo "${GHCR_IMAGE}:latest"
+              [[ "$SYNC_DEV" == true ]] && echo "${GHCR_IMAGE}:dev"
             fi
             echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
             [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${GHCR_IMAGE}:${VERSION}"
@@ -276,7 +408,7 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
           TAG_ARGS=()
           while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
           REFS=()
@@ -287,7 +419,7 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
           TAG_ARGS=()
           while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
           REFS=()
@@ -317,6 +449,7 @@ jobs:
     name: Cascade verified release
     if: needs.python-release.outputs.release_ready == 'true'
     needs:
+      - candidate
       - python-release
       - docker-release
     permissions:
@@ -327,5 +460,6 @@ jobs:
       upstream_package: archivebox
       upstream_version: ${{ needs.python-release.outputs.version }}
       upstream_artifact_name: python-distributions
+      upstream_run_id: ${{ needs.candidate.outputs.artifact_run_id }}
     secrets:
       release_token: ${{ secrets.RELEASE_GH_TOKEN }}

.github/workflows/screenshots.yml

@@ -0,0 +1,71 @@
+name: Capture publicsite screenshots
+
+on:
+  push:
+    branches:
+      - dev
+    paths:
+      - archivebox/**
+      - bin/collect_ui_screenshots.*
+      - bin/generate_ui_screenshot_gallery.py
+      - bin/take_screenshot.js
+      - docs/Screenshots.md
+      - pyproject.toml
+      - uv.lock
+      - .github/workflows/screenshots.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.event_name == 'workflow_dispatch' && format('publicsite-captures-manual-{0}', github.run_id) || 'publicsite-captures' }}
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          fetch-depth: 1
+
+      - name: Setup Python and uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: false
+
+      - name: Install ArchiveBox and screenshot dependencies
+        run: uv sync --no-cache --frozen
+
+      - name: Build UI screenshot gallery
+        run: bin/collect_ui_screenshots.sh
+        env:
+          BASE_URL: http://archivebox.localhost:5797
+
+      - name: Verify gallery was generated from this exact revision
+        run: |
+          uv run --no-cache --no-sync python - <<'PY'
+          import json
+          import os
+          import re
+          import tomllib
+          from pathlib import Path
+
+          provenance = json.loads(Path("publicsite/screenshots/build.json").read_text())
+          version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
+          assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance
+          assert provenance["version"] == version, provenance
+          assert provenance["capture_count"] >= 3 * 35, provenance
+          assert len(provenance["files"]) == provenance["capture_count"], provenance
+          assert all(not re.match(r"^\d+-", name) for name in provenance["files"]), provenance
+          PY
+
+      - name: Save validated screenshots
+        uses: actions/upload-artifact@v4
+        with:
+          name: site-screenshots
+          path: publicsite/screenshots
+          if-no-files-found: error
+          retention-days: 90

.github/workflows/test-parallel.yml

@@ -2,6 +2,12 @@ name: Parallel Tests
 
 on:
   workflow_call:
+    inputs:
+      test_file:
+        description: Run one discovered test file; empty runs the complete matrix
+        required: false
+        default: ""
+        type: string
 
 env:
   PYTHONIOENCODING: utf-8
@@ -58,10 +64,15 @@ jobs:
 
       - name: Discover every test exactly once
         id: discover
+        env:
+          TEST_FILE: ${{ inputs.test_file }}
         run: |
           set -Eeuo pipefail
           mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
           test "${#output[@]}" -eq 2
+          if [[ -n "$TEST_FILE" ]]; then
+            output[1]="$("$JQ_BINARY" -ce '[.[] | select(.paths == [env.TEST_FILE])] | select(length == 1)' <<< "${output[1]}")"
+          fi
           echo "tests=${output[1]}" >> "$GITHUB_OUTPUT"
           echo "${output[0]}"
           "$JQ_BINARY" -e 'length > 0 and length <= 256 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}"
@@ -132,6 +143,9 @@ jobs:
             extra_args=(--extra "$TEST_EXTRA")
           fi
           uv sync --no-cache --locked --dev "${extra_args[@]}"
+      - name: Install Playwright Chromium for browser tests
+        if: contains(matrix.test.paths_arg, 'test_proof_previews') || contains(matrix.test.paths_arg, 'test_snapshot_output_stacks')
+        run: uv run --no-cache --no-sync --no-sources playwright install chromium
 
       - name: Run ${{ matrix.test.name }}
         env:

.gitignore

@@ -68,3 +68,5 @@ ArchiveBox.conf
 
 # Local Claude Code task locks (not project state)
 .claude/scheduled_tasks.lock
+
+/_site/

AGENTS.md

@@ -84,4 +84,4 @@ uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_ta
 uv run prek run --all-files
 ```
 
-Releases are published only by `.github/workflows/release.yml` after the complete `dev` CI workflow succeeds. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.
+Releases are published only by `.github/workflows/release.yml` after complete CI succeeds for the exact commit. Pushes to `main` prepare stable patch releases automatically; `dev` prepares release candidates. Version selection and dependency repinning are owned by CI and the monorepo coordinator. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.

Dockerfile

@@ -1,8 +1,38 @@
 # syntax=docker/dockerfile:1.7
 
 # Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
-# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
-# ArchiveBox owns sonic, supervisor, Django, and the app runtime.
+# WHY: abx-dl owns Python, Node, Chromium, and extraction dependencies so the
+# standalone downloader stays independent of ArchiveBox's server features.
+# ArchiveBox-only dependencies (Sonic, OpenCode, supervisor, Django) belong in
+# the layers added by this Dockerfile, never in the shared abx-dl base image.
+# REQUIRED INSTALLATION CONTRACT — do not work around failures:
+# 1. IMAGE CONSTRUCTION: preinstall ALL plugins supported by runtime="archivebox"
+#    and ALL their required_binaries, regardless of default enabled state.
+#    Inherit the complete downloader dependencies from the tested abx-dl image;
+#    install every remaining ArchiveBox-only dependency in this image's layers,
+#    including OpenCode and Sonic. Never put server dependencies in abx-dl.
+#    Use the normal plugin dependency pipeline: install command -> config.json
+#    required_binaries -> abxpkg. Never add manual apt/pip/npm installs to hide
+#    failures in that pipeline. OS/interpreter bootstrap is a separate concern,
+#    not a second hand-maintained list of plugin dependencies.
+# 2. FINISHED IMAGE: archivebox install and the full plugin installation check
+#    must discover everything already present and be NO-OPS offline as UID 911.
+#    No downloads, package installs, sudo, or root repair may be needed to enable
+#    ANY plugin or start a new crawl. Never run verification as root or manually
+#    preinstall packages before it. If the inherited image lacks a downloader
+#    dependency, fix/release abx-dl and consume that tested image; do not patch
+#    over the missing base dependency here or repair deployed containers by SSH.
+# 3. NON-CONTAINER CI: exercise real runtime auto-installation through the same
+#    CLI/required_binaries/abxpkg path users invoke. GitHub's runner has
+#    passwordless sudo; abxpkg must escalate automatically when needed. Never
+#    manually preinstall dependencies or wrap the tested command in sudo to make
+#    a failure disappear. This is distinct from the finished-image no-op test.
+# 4. SIZE: the complete compressed image budget is 1025 MiB. Preserve ALL
+#    user-facing functionality. Only mechanical/internal
+#    optimizations are allowed. Never remove dependencies, disable features or
+#    plugins, weaken assertions, or skip checks to meet a size limit. If the
+#    complete image exceeds the configured limit, LET CI FAIL and explicitly
+#    ask the maintainer what to do. Do not raise the limit unilaterally.
 # Build abx-dl first, then point this file at it:
 #   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
 #       --build-context abxbus=../abxbus \
@@ -10,10 +40,10 @@
 #       --build-context abx-plugins=../abx-plugins \
 #       -t archivebox/abx-dl:dev
 #   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.282 \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:dev \
 #       -t archivebox:multistage
 
-ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.282
+ARG ABX_DL_IMAGE=archivebox/abx-dl:main
 
 FROM archivebox/sonic:1.4.9 AS sonic
 FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
@@ -47,6 +77,7 @@ ENV ARCHIVEBOX_USER=archivebox \
     IN_DOCKER=True
 
 ENV CODE_DIR=/app \
+    ABX_RUNTIME=archivebox \
     DATA_DIR=/data \
     CONFIG_DIR=/opt/archivebox \
     ABXPKG_LIB_DIR=/opt/archivebox/lib \
@@ -250,6 +281,21 @@ RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777
     && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
     && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
 
+# Install the complete ArchiveBox catalog, including disabled server plugins.
+# Explicit plugin names make installation independent of enabled defaults.
+RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-opencode-$TARGETARCH \
+    chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" /var/tmp/abxpkg-cache \
+    && chmod 1777 /var/tmp/abxpkg-cache \
+    && export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
+    && env ABXPKG_TMP_CACHE_DIR=/var/tmp/abxpkg-cache XDG_CACHE_HOME=/var/tmp/abxpkg-cache \
+        setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install $ABX_DOCKER_PLUGINS \
+    # pnpm includes musl variants that Debian's glibc runtime cannot use.
+    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules" -type l -name 'opencode-linux-*-musl' -delete \
+    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.pnpm" -maxdepth 1 -type d -name 'opencode-linux-*-musl@*' -exec rm -rf {} +
+RUN --network=none export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups \
+    bash -c '"$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version && abx-dl install $ABX_DOCKER_PLUGINS'
+
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
     && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
@@ -282,10 +328,10 @@ RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully
 
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
-EXPOSE 8000
+EXPOSE 5797
 
 HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
+    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:5797/health/' | grep -q 'OK'
 
 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]
+CMD ["archivebox", "server", "--init", "0.0.0.0:5797"]

README.md

@@ -73,15 +73,15 @@ mkdir -p ~/archivebox/data && cd ~/archivebox
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
 docker compose pull
 docker compose up -d --wait                                                # initializes new collections automatically
-# open http://admin.archivebox.localhost:8000 to finish setup
+# open http://admin.archivebox.localhost:5797 to finish setup
 # docker compose run --rm archivebox add 'https://example.com'
 # docker compose run --rm archivebox help
 <br/>
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -d --name archivebox -v "$PWD:/data" -p 8000:8000 archivebox/archivebox:dev
-# open http://admin.archivebox.localhost:8000 to finish setup
+docker run -d --name archivebox -v "$PWD:/data" -p 5797:5797 archivebox/archivebox:dev
+# open http://admin.archivebox.localhost:5797 to finish setup
 # docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
 # docker run -it -v $PWD:/data archivebox/archivebox:dev help
 <br/>
@@ -93,14 +93,14 @@ archivebox init
 archivebox install
 # archivebox add 'https://example.com'
 # archivebox help
-# archivebox server 0.0.0.0:8000
+# archivebox server 0.0.0.0:5797
 <br/>
 <br/>
 # Option D: Or use the uv install shortcut for Option C
 curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
-<sub>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sub><br/>
+<sub>Open <a href="http://web.archivebox.localhost:5797"><code>http://web.archivebox.localhost:5797</code></a> for the public UI and <a href="http://admin.archivebox.localhost:5797"><code>http://admin.archivebox.localhost:5797</code></a> for the admin UI ➡️</sub><br/>
 <sub>Set <code>BASE_URL</code> to change the public base domain. The default <code>auto</code> mode uses <code>web.</code> and <code>admin.</code> subdomains on <code>*.localhost</code>, but one host for ordinary DNS names. <code>BIND_ADDR</code> only controls the local listen address.</sub>
 </details>
 <br/>
@@ -129,10 +129,10 @@ curl -fsSL 'https://get.archivebox.io' | bash
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
 - [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
-- Advanced users: support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
-- Planned: support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
+- Support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
+- Support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [infiniscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
 
 <br/>
 
@@ -151,6 +151,17 @@ curl -fsSL 'https://get.archivebox.io' | bash
 
 #### ✳️&nbsp; Easy Setup
 
+<details>
+<summary><b><img src="https://app.archivebox.io/docs/icons/apple.svg" alt="Apple" height="28px" align="top"/> <code>ArchiveBox Server.app</code></b> (macOS)</summary>
+<br/>
+<ol>
+<li><a href="https://github.com/ArchiveBox/ios-archivebox/releases">Download <code>ArchiveBox Server.app</code></a>, move it to Applications, and open it.</li>
+<li>Set up an admin account and generate an API Key.</li>
+<li>Use the Web UI or <a href="https://app.archivebox.io/"><code>ArchiveBox.app</code></a>, and optionally connect REST API clients, the <a href="https://app.archivebox.io/">mobile app</a>, the <a href="https://extension.archivebox.io/">browser extension</a>, and other integrations to your server using your API Key.</li>
+</ol>
+<br/>
+</details>
+
 <details>
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker-compose</code></b>  (macOS/Linux) &nbsp; <b>👈&nbsp; recommended</b> &nbsp; <i>(click to expand)</i></summary>
 <br/>
@@ -166,7 +177,7 @@ docker compose pull
 <li>Start the server, which initializes a new collection automatically.
 <pre lang="bash"><code style="white-space: pre-line">docker compose up -d --wait
 </code></pre></li>
-<li>Open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
+<li>Open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:5797/admin/">http://admin.archivebox.localhost:5797/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
 <pre lang="bash"><code style="white-space: pre-line">
 # run CLI commands inside the server container started above
 docker compose exec archivebox archivebox add 'https://example.com'
@@ -187,10 +198,10 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
 <li>Create a new empty directory and start the server, which initializes the collection automatically (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -d --name archivebox -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
+docker run -d --name archivebox -v $PWD:/data -p 5797:5797 archivebox/archivebox:dev
 </code></pre>
 </li>
-<li>Open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
+<li>Open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:5797/admin/">http://admin.archivebox.localhost:5797/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
 <pre lang="bash"><code style="white-space: pre-line">
 # completely optional, CLI can always be used without running a server
 # docker exec archivebox archivebox [subcommand] [--help]
@@ -245,8 +256,8 @@ archivebox init     # initialize a new collection
 archivebox install  # install all the runtime dependencies (e.g. chrome, single-file, yt-dlp, etc.)
 </code></pre>
 </li>
-<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
-<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
+<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:5797/admin/">http://admin.archivebox.localhost:5797/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
+<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:5797
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
@@ -281,8 +292,8 @@ archivebox add 'https://example.com'
 </code></pre>
 <br/>
 </li>
-<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
-<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
+<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:5797/admin/">http://admin.archivebox.localhost:5797/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
+<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:5797
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
@@ -314,8 +325,8 @@ archivebox init
 archivebox install
 </code></pre>
 </li>
-<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
-<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
+<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:5797/admin/">http://admin.archivebox.localhost:5797/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
+<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:5797
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
@@ -354,17 +365,12 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 <br/>
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
-<li>Download a binary release for your OS or build the native app from source<br/>
-<ul>
-<li>macOS: <a href="https://github.com/ArchiveBox/ArchiveBox/releases/download/v0.6.2/Electron-ArchiveBox-macOS-x64-0.6.2.app.zip" download><code>ArchiveBox.app.zip</code></a></li>
-<li>Linux: <code>ArchiveBox.deb</code> (alpha: <a href="https://github.com/ArchiveBox/electron-archivebox#quickstart">build manually</a>)</li>
-<li>Windows: <code>ArchiveBox.exe</code> (beta: <a href="https://github.com/ArchiveBox/electron-archivebox#quickstart">build manually</a>)</li>
-</ul>
-</li>
+<li><a href="https://electron.archivebox.io/#get-started">Download the latest installer for Windows, Linux, or Mac</a>.</li>
+<li>Install and open ArchiveBox Desktop, then create your local administrator account to start archiving.</li>
 </ol>
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/575ef92f-bb3e-4a7c-a4ba-986c1fd76ecf" width="320px">
 <br/>
-<i>✨ Alpha (contributors wanted!)</i>: for more info, see the: <a href="https://github.com/ArchiveBox/electron-archivebox">Electron ArchiveBox</a> repo.
+<a href="https://electron.archivebox.io/">Desktop app guide</a> · <a href="https://electron.archivebox.io/screenshots/">Screenshots</a>
 <br/>
 </details>
 
@@ -533,16 +539,16 @@ find ./archive/users -path '*/snapshots/*/*/*/index.html'  # inspect snapshot da
 <summary><b>🖥&nbsp; Web UI & API Usage</b></summary>
 <pre lang="bash"><code style="white-space: pre-line">
 # Start the server on bare metal (uv/apt/brew):
-archivebox server 0.0.0.0:8000                 # start the server
+archivebox server 0.0.0.0:5797                 # start the server
 <br/>
 # Or with Docker Compose:
 docker compose up                              # start the server
 <br/>
 # Or with a Docker container:
-docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
+docker run -v $PWD:/data -it -p 5797:5797 archivebox/archivebox:dev
 </code></pre>
 
-<sup>Open <a href="http://admin.archivebox.localhost:8000/admin/"><code>http://admin.archivebox.localhost:8000/admin/</code></a> to create the first admin and finish web setup. Use <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI. ➡️</sup><br/>
+<sup>Open <a href="http://admin.archivebox.localhost:5797/admin/"><code>http://admin.archivebox.localhost:5797/admin/</code></a> to create the first admin and finish web setup. Use <a href="http://web.archivebox.localhost:5797"><code>http://web.archivebox.localhost:5797</code></a> for the public UI. ➡️</sup><br/>
 <sup>Advanced: <code>archivebox manage createsuperuser</code> remains available for creating accounts from the CLI.</sup><br/>
 <sup>Set <code>BASE_URL</code> to change the public base domain. The default <code>auto</code> mode uses <code>web.</code> and <code>admin.</code> subdomains on <code>*.localhost</code>, but one host for ordinary DNS names. <code>BIND_ADDR</code> only controls the local listen address.</sup>
 <br/><br/>
@@ -559,6 +565,22 @@ docker compose run --rm archivebox config --set PERMISSIONS=public
 
 # restart the server to apply any config changes
 </code></pre>
+
+Snapshot caching follows each snapshot's saved `PERMISSIONS`, including inherited defaults and snapshot overrides. Public content can be cached by browsers and shared proxies/CDNs; private and unlisted content uses `Cache-Control: private` so only the user's browser can cache it. Both retain normal artifact cache lifetimes and validators for fast repeat viewing. Authentication handoffs are not stored. Changing permissions does not revoke copies already cached under the previous policy; purge any affected shared cache when needed.
+</details>
+
+<br/>
+
+<details>
+<summary><b><img src="https://app.archivebox.io/docs/icons/apple.svg" alt="Apple" height="22px" align="top"/> <img src="https://cdn.jsdelivr.net/gh/devicons/devicon@v2.17.0/icons/windows11/windows11-original.svg" alt="Windows" height="22px" align="top"/> 🐧&nbsp; Desktop / Mobile / Extension Usage</b></summary>
+<br/>
+
+- 📱 [<code>ArchiveBox</code> iOS Mobile App](https://app.archivebox.io/)
+- 🖥️ [<code>ArchiveBox.app</code> macOS Desktop App](https://app.archivebox.io/)
+- 🧩 [ArchiveBox Browser Extension](https://extension.archivebox.io/)
+- 💻 [Linux/Windows Desktop App (Requires Docker Desktop)](https://electron.archivebox.io/)
+
+<br/>
 </details>
 
 <br/>
@@ -614,7 +636,10 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
 
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ff20d251-5347-4b85-ae9b-83037d0ac01e" height="28px"/> <b>From the official <a href="https://github.com/ArchiveBox/archivebox-extension">ArchiveBox Browser Extension</a></b>  
-  <i>Provides realtime archiving of browsing history or selected pages from Chrome/Chromium/Firefox browsers.</i>
+  <i>Provides realtime archiving of browsing history or selected pages from Chrome/Chromium/Firefox/Safari browsers.</i>
+
+- <img src="https://app.archivebox.io/App/AppIcon.icon/Assets/ArchiveBox.png" alt="ArchiveBox app icon" height="28px"/> <b>From the official <a href="https://app.archivebox.io/">ArchiveBox macOS Desktop App or iOS Mobile App</a></b><br/>
+  <i>Save directly from the Share menu or Siri/Shortcuts to your server.</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8" height="22px"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files  
   <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file).</i>
@@ -980,14 +1005,11 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active
 
 
 <ul>
-<li>Set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#curl_user_agent"><code>CHROME_USER_AGENT</code>, <code>WGET_USER_AGENT</code>, <code>CURL_USER_AGENT</code></a> to impersonate a real browser (by default, ArchiveBox reveals that it's a bot when using the default user agent settings)</li>
-<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">importing a dedicated browser profile into a persona</a></li>
+<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
 <li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
 
-In the future we plan on adding support for running JS scripts during archiving to block ads, cookie popups, modals, and fix other issues. Follow here for progress: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/51">Issue #51</a>.
-
 </details>
 <br/>
 
@@ -1318,10 +1340,10 @@ archivebox init
 archivebox install         # detect and install all extractor dependencies
 
 # Run the development server w/ autoreloading (but no bg workers)
-archivebox server --debug --reload 0.0.0.0:8000
+archivebox server --debug --reload 0.0.0.0:5797
 
 # Run the production server (with bg workers but no autoreloading)
-archivebox server 0.0.0.0:8000
+archivebox server 0.0.0.0:5797
 ```
 
 #### 2. Option B: Build the docker container and use that for development instead
@@ -1333,10 +1355,10 @@ archivebox server 0.0.0.0:8000
 ./bin/build_docker.sh dev
 
 # Run the development server w/ autoreloading (but no bg workers)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init --debug --reload 0.0.0.0:8000
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 5797:5797 archivebox/archivebox:dev server --init --debug --reload 0.0.0.0:5797
 
 # Run the production server (with bg workers but no autoreloading)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 5797:5797 archivebox/archivebox:dev server --init
 
 # (remove the --reload flag and add the --nothreading flag when profiling with the django debug toolbar)
 # When using --reload, make sure any files you create can be read by the user in the Docker container, eg with 'chmod a+rX'.
@@ -1358,11 +1380,11 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 archivebox config --set DEBUG=True
 
 # OR you can run a dev server with DEBUG=True in a few ways:
-archivebox server --debug --reload 0.0.0.0:8000
+archivebox server --debug --reload 0.0.0.0:5797
 # or
-archivebox server --debug 0.0.0.0:8000
+archivebox server --debug 0.0.0.0:5797
 # or
-env DEBUG=True daphne -b 0.0.0.0 -p 8000 archivebox.core.asgi:application
+env DEBUG=True daphne -b 0.0.0.0 -p 5797 archivebox.core.asgi:application
 ```
 
 https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running
@@ -1397,7 +1419,7 @@ services:
 
 # or with plain Docker:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
-docker run -it -v $PWD:/data -p 8000:8000 archivebox:dev
+docker run -it -v $PWD:/data -p 5797:5797 archivebox:dev
 
 # or with uv:
 uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'

archivebox/api/v1_core.py

@@ -247,7 +247,7 @@ def get_archiveresult(request: HttpRequest, archiveresult_id: str):
 
 def _normalize_uploaded_archiveresult_plugin(plugin: str) -> str:
     normalized = str(plugin or "").strip().strip("/")
-    if not ARCHIVERESULT_UPLOAD_PLUGIN_RE.fullmatch(normalized):
+    if normalized in (".", "..") or not ARCHIVERESULT_UPLOAD_PLUGIN_RE.fullmatch(normalized):
         raise HttpError(400, "Invalid ArchiveResult plugin name")
     return normalized
 
@@ -386,6 +386,7 @@ def _write_archiveresult_files(
     existing_output_files: dict[str, dict[str, Any]] | None = None,
     allow_empty: bool = False,
 ) -> dict[str, dict[str, Any]]:
+    plugin_name = _normalize_uploaded_archiveresult_plugin(plugin_name)
     files = _get_archiveresult_upload_files(request, allow_empty=allow_empty)
     output_paths = _get_archiveresult_upload_form_values(request, "output_paths", "output_path")
     mime_types = _get_archiveresult_upload_form_values(request, "mime_types", "mime_type")

archivebox/api/v1_personas.py

@@ -2,7 +2,7 @@ __package__ = "archivebox.api"
 
 import json
 from datetime import datetime
-from typing import Any
+from typing import Any, Literal
 from uuid import UUID
 
 from django.db.models import Q
@@ -23,6 +23,7 @@ class PersonaBrowserSettingsSchema(Schema):
     user_agent: str = ""
     viewport_size: str = ""
     viewport_device_scale_factor: float | None = None
+    color_scheme: Literal["", "light", "dark"] = ""
     language: str = ""
     timezone: str = ""
     geolocation: dict[str, Any] | None = None
@@ -94,6 +95,8 @@ def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrows
         )
     if settings.viewport_device_scale_factor is not None:
         config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor
+    if "color_scheme" in settings.model_fields_set:
+        config["BROWSER_COLOR_SCHEME"] = settings.color_scheme
     if settings.language:
         config["BROWSER_LANGUAGE"] = settings.language
     if settings.timezone:
@@ -150,12 +153,12 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     persona.ensure_dirs()
 
     cookies_written = False
-    if payload.cookies_txt.strip():
+    if "cookies_txt" in payload.model_fields_set:
         (persona.path / "cookies.txt").write_text(payload.cookies_txt)
         cookies_written = True
 
     auth_written = False
-    if payload.auth_json:
+    if "auth_json" in payload.model_fields_set:
         (persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n")
         auth_written = True
 

archivebox/cli/archivebox_add.py

@@ -6,6 +6,7 @@ __command__ = "archivebox add"
 
 import sys
 import os
+import signal
 from pathlib import Path
 
 from typing import Any, TYPE_CHECKING
@@ -227,6 +228,10 @@ def add(
 
         assert command is not None
         exit_code = 0
+        interactive_interrupts = sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())
+
+        def interrupt_hook() -> None:
+            raise KeyboardInterrupt
 
         def crawl_is_complete() -> bool:
             crawl.refresh_from_db(fields=["status"])
@@ -234,7 +239,13 @@ def add(
 
         try:
             try:
-                with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
+                with (
+                    foreground_shutdown_signals(
+                        first_signal_message=None,
+                        interrupt_handlers={signal.SIGINT: interrupt_hook},
+                    ),
+                    foreground_parent_watchdog(),
+                ):
                     while True:
                         standby = standby_until_foreground_runner_needed(
                             command,
@@ -246,7 +257,7 @@ def add(
                         exit_code = run_runner_worker(
                             ["--crawl-id", str(crawl.id)],
                             name=f"worker_runner_add_{os.getpid()}",
-                            interactive_interrupts=True,
+                            interactive_interrupts=interactive_interrupts,
                             config=get_config(crawl=crawl),
                         )
                         crawl.refresh_from_db(fields=["status", "retry_at"])
@@ -311,9 +322,13 @@ def add(
 
             admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
 
-            print("\n[bold]crawl output saved to:[/bold]")
-            print(f"  {rel_output_str}")
-            print(f"  {admin_url}")
+            if snapshots_count:
+                print("\n[bold]snapshot output saved to:[/bold]")
+                for snapshot in crawl.snapshot_set.order_by("created_at").iterator():
+                    click.echo(f"  {snapshot.output_dir}")
+            print("\n[bold]crawl logs and setup:[/bold]")
+            click.echo(f"  {rel_output_str}")
+            click.echo(f"  {admin_url}")
             print(f"\n[bold]total urls snapshotted:[/bold] {snapshots_count}")
             print(f"[bold]total size:[/bold] {total_size}")
             print(f"[bold]total time:[/bold] {duration_str}")

archivebox/cli/archivebox_help.py

@@ -68,7 +68,7 @@ def help() -> None:
     [blue]docker compose run[/blue] [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
 
     [grey53]# using Docker:[/grey53]
-    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 8000:8000[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
+    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 5797:5797[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
 """
         if IN_DOCKER
         else ""
@@ -109,7 +109,7 @@ def help() -> None:
     [dark_green]archivebox[/dark_green] [green]add[/green] --depth=1 "https://example.com/some/page"
     [dark_green]archivebox[/dark_green] [green]list[/green] --sort=timestamp --csv=timestamp,downloaded_at,url,title
     [dark_green]archivebox[/dark_green] [green]schedule[/green] --every=day --depth=1 "https://example.com/some/feed.rss"
-    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:8000[/blue]                [grey53]# Start the Web UI / API server[/grey53]
+    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:5797[/blue]                [grey53]# Start the Web UI / API server[/grey53]
 """
         print(
             Panel(

archivebox/cli/archivebox_init.py

@@ -134,7 +134,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     print("    > Skipping orphan snapshot import during init.")
     print()
     print("    [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:")
-    print("        archivebox update")
+    print("        archivebox update --rescan --migrate-only")
 
     print("\n[green]----------------------------------------------------------------------[/green]")
 

archivebox/cli/archivebox_persona.py

@@ -109,6 +109,7 @@ def create_personas(
     profile: str | None = None,
     source: str | None = None,
     browser_binary: str | None = None,
+    permissions: str | None = None,
 ) -> int:
     """
     Create Personas from names.
@@ -162,7 +163,10 @@ def create_personas(
             rprint(f'[red]Invalid persona name "{name}": {error_msg}[/red]', file=sys.stderr)
             continue
 
-        persona, created = Persona.objects.get_or_create(name=name)
+        persona, created = Persona.objects.get_or_create(
+            name=name,
+            defaults={"config": {"PERMISSIONS": permissions}} if permissions else {},
+        )
 
         if created:
             persona.ensure_dirs()
@@ -441,7 +445,11 @@ def main():
 
 @main.command("create")
 @click.argument("names", nargs=-1)
-@click.option("--import", "import_from", help="Import from chrome, chromium, brave, edge, a persona directory, or a live CDP URL")
+@click.option(
+    "--import",
+    "import_from",
+    help="Import from chrome, chrome-beta, chromium, brave, edge, a persona directory, or a live CDP URL",
+)
 @click.option("--source", help="Source browser user-data directory or exact profile path (including Docker mounts)")
 @click.option(
     "--browser-binary",
@@ -449,9 +457,30 @@ def main():
     help="Source Chromium browser executable; required for other Chromium-based browsers",
 )
 @click.option("--profile", help="Profile directory name under the user data dir (e.g. Default, Profile 1)")
-def create_cmd(names: tuple, import_from: str | None, profile: str | None, source: str | None, browser_binary: str | None):
+@click.option(
+    "--permissions",
+    type=click.Choice(["public", "unlisted", "private"]),
+    help="Visibility for a newly created persona (existing personas are unchanged)",
+)
+def create_cmd(
+    names: tuple,
+    import_from: str | None,
+    profile: str | None,
+    source: str | None,
+    browser_binary: str | None,
+    permissions: str | None,
+):
     """Create Personas, optionally importing from a browser profile."""
-    sys.exit(create_personas(names, import_from=import_from, profile=profile, source=source, browser_binary=browser_binary))
+    sys.exit(
+        create_personas(
+            names,
+            import_from=import_from,
+            profile=profile,
+            source=source,
+            browser_binary=browser_binary,
+            permissions=permissions,
+        ),
+    )
 
 
 @main.command("list")

archivebox/cli/archivebox_run.py

@@ -54,12 +54,9 @@ RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
 
 
 def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
-    # A supervised `archivebox run --daemon` is intentionally a disposable
-    # child. If it receives SIGINT/SIGTERM directly, exit with the conventional
-    # signal status so supervisord treats it as an unexpected worker death and
-    # restarts only the runner. The parent `archivebox server` owns supervisord
-    # shutdown and must not be pulled down by a killed daemon worker.
-    os._exit(128 + int(sig))
+    # A supervised daemon must exit non-zero so supervisord restarts it, while
+    # still unwinding the active crawl's hooks and persisted process state.
+    raise SystemExit(128 + sig.value)
 
 
 def process_stdin_records() -> int:
@@ -285,19 +282,26 @@ def run_runner(
         # future ownership lease cannot hide it from the unified scheduler.
         if crawl is not None:
             crawl.update_and_requeue(retry_at=now, refresh=False)
-    # Only a foreground `archivebox add` gets the interactive "abort current
-    # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
-    # orchestrators should shut down immediately and cleanly on the first signal.
-    interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
+    # Interactivity follows terminal ownership, not the spelling of the command.
+    # A supervised add worker has pipes and receives terminal intent/answers from
+    # its parent; direct run owns the terminal itself. Daemons and redirected
+    # commands must never wait for a person who cannot answer their prompt.
+    # Trust the explicit terminal-owner flag, not Process.root: when add borrows
+    # a server-owned supervisord, its worker's OS ancestry still leads to server.
+    # Gating on root=ADD silently made the first Ctrl+C abort after takeover.
+    interactive_interrupts = not daemon and (
+        (sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())) or os.environ.get("ARCHIVEBOX_INTERACTIVE_INTERRUPTS") == "1"
+    )
     if daemon:
         os.environ[RUNNER_DAEMON_ENV] = "1"
 
+    shutdown_state = None
     try:
         with (
             foreground_shutdown_signals(
                 on_signal=_exit_daemon_runner_on_signal if daemon else None,
                 raise_on_first_signal=not daemon,
-            ),
+            ) as shutdown_state,
             foreground_parent_watchdog(enabled=not daemon),
         ):
             run_pending_crawls(
@@ -306,10 +310,16 @@ def run_runner(
                 maintenance_only=maintenance_only,
                 interactive_interrupts=interactive_interrupts,
             )
+        if daemon and shutdown_state.signal_name:
+            return 128 + signal.Signals[shutdown_state.signal_name].value
         return 0
     except KeyboardInterrupt:
-        return 0
+        if daemon and shutdown_state and shutdown_state.signal_name:
+            return 128 + signal.Signals[shutdown_state.signal_name].value
+        return 130
     except asyncio.CancelledError as e:
+        if daemon and shutdown_state and shutdown_state.signal_name:
+            return 128 + signal.Signals[shutdown_state.signal_name].value
         if daemon:
             rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
             return 1
@@ -362,11 +372,11 @@ def main(
             ):
                 sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
         except KeyboardInterrupt:
-            sys.exit(0)
+            sys.exit(130)
 
     with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
         if snapshot_id:
-            sys.exit(run_snapshot_worker(snapshot_id))
+            sys.exit(run_snapshot_worker(snapshot_id, daemon=daemon))
 
         if binary_id:
             try:
@@ -386,7 +396,7 @@ def main(
         if crawl_id:
             sys.exit(
                 run_runner(
-                    daemon=False,
+                    daemon=daemon,
                     crawl_id=crawl_id,
                     maintenance_only=maintenance_only,
                 ),
@@ -401,7 +411,7 @@ def main(
             sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
 
 
-def run_snapshot_worker(snapshot_id: str) -> int:
+def run_snapshot_worker(snapshot_id: str, *, daemon: bool = False) -> int:
     from archivebox.config import CONSTANTS
     from archivebox.core.takeover_util import enter_single_runner_gate
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
@@ -416,6 +426,7 @@ def run_snapshot_worker(snapshot_id: str) -> int:
         return 0
 
     snapshot = None
+    interactive_interrupts = not daemon and sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             for _ in range(10):
@@ -424,7 +435,7 @@ def run_snapshot_worker(snapshot_id: str) -> int:
                     snapshot.update_and_requeue(retry_at=timezone.now())
                 elif snapshot.retry_at > timezone.now():
                     break
-                if not run_due_snapshot(snapshot, lock_seconds=60):
+                if not run_due_snapshot(snapshot, lock_seconds=60, interactive_interrupts=interactive_interrupts):
                     break
         return 0
     except KeyboardInterrupt:
@@ -437,7 +448,7 @@ def run_snapshot_worker(snapshot_id: str) -> int:
                 snapshot.update_and_requeue(retry_at=timezone.now())
         except Exception:
             pass
-        return 0
+        return 130
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
         import traceback

archivebox/cli/archivebox_server.py

@@ -46,13 +46,13 @@ def _split_bind_spec(spec: str) -> tuple[str, str]:
     """Split a ``host:port`` / ``host`` / ``port`` spec into ``(host, port)``.
 
     The empty strings stand in for "not provided"; the caller fills in
-    defaults. Bracketed IPv6 literals like ``[::1]:8000`` are handled.
+    defaults. Bracketed IPv6 literals like ``[::1]:5797`` are handled.
     """
     spec = (spec or "").strip()
     if not spec:
         return "", ""
     if spec.startswith("["):
-        # Bracketed IPv6: ``[::1]`` or ``[::1]:8000``
+        # Bracketed IPv6: ``[::1]`` or ``[::1]:5797``
         end = spec.find("]")
         if end == -1:
             return spec, ""  # malformed; let validator reject it
@@ -77,11 +77,11 @@ def _parse_and_validate_bind_spec(spec: str) -> tuple[str, str]:
     (normalized to ``127.0.0.1``). Bare hostnames are rejected because the
     bind address feeds Daphne, which has to listen on a numeric address;
     public hostnames belong in ``BASE_URL`` instead. Empty values fall back
-    to ``127.0.0.1`` / ``8000``.
+    to ``127.0.0.1`` / ``5797``.
     """
     raw_host, raw_port = _split_bind_spec(spec)
     host = raw_host.strip()
-    port = (raw_port or "").strip() or "8000"
+    port = (raw_port or "").strip() or "5797"
 
     if host == "" or host.lower() == "localhost":
         host = "127.0.0.1"
@@ -198,7 +198,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
             "[yellow]    routing cannot work against an IP address. Set BASE_URL explicitly, e.g.[/yellow]",
         )
         print(
-            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
+            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:5797[/yellow]",
         )
         if config.USES_SUBDOMAIN_ROUTING:
             print(
@@ -221,7 +221,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
             "[yellow]    a reverse proxy / ingress / public hostname, e.g.[/yellow]",
         )
         print(
-            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
+            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:5797[/yellow]",
         )
         print()
 
@@ -254,7 +254,7 @@ def server(
         print()
 
     # First non-empty positional arg is the bind spec; otherwise inherit from
-    # config (which defaults to "127.0.0.1:8000"). _parse_and_validate_bind_spec
+    # config (which defaults to "127.0.0.1:5797"). _parse_and_validate_bind_spec
     # hard-errors on hostnames so the rest of the server can assume a numeric
     # bind host.
     bind_spec = next((arg for arg in runserver_args if arg), "")

archivebox/cli/archivebox_snapshot.py

@@ -44,6 +44,7 @@ SNAPSHOT_LIST_CHUNK_SIZE = 5000
 
 def iter_snapshot_json(queryset: QuerySet) -> Iterator[dict[str, object]]:
     from archivebox.config import VERSION
+    from archivebox.config.common import redact_sensitive_config
     from archivebox.core.models import SnapshotTag
 
     fields = (
@@ -53,6 +54,9 @@ def iter_snapshot_json(queryset: QuerySet) -> Iterator[dict[str, object]]:
         "title",
         "bookmarked_at",
         "created_at",
+        "modified_at",
+        "config",
+        "notes",
         "timestamp",
         "depth",
         "status",
@@ -80,6 +84,9 @@ def iter_snapshot_json(queryset: QuerySet) -> Iterator[dict[str, object]]:
                 "tags": ",".join(sorted(tags_by_snapshot[row["id"]])),
                 "bookmarked_at": row["bookmarked_at"].isoformat() if row["bookmarked_at"] else None,
                 "created_at": row["created_at"].isoformat() if row["created_at"] else None,
+                "modified_at": row["modified_at"].isoformat() if row["modified_at"] else None,
+                "config": redact_sensitive_config(row["config"]),
+                "notes": row["notes"],
                 "timestamp": row["timestamp"],
                 "depth": row["depth"],
                 "status": row["status"],

archivebox/cli/archivebox_update.py

@@ -7,6 +7,7 @@ import os
 import asyncio
 import shlex
 import time
+from itertools import islice
 
 from typing import TYPE_CHECKING, Any
 from collections.abc import Iterable
@@ -52,17 +53,20 @@ def _get_search_indexing_plugins() -> list[str]:
 def _build_filtered_snapshots_queryset(
     **kwargs,
 ):
-    from archivebox.core.models import Snapshot
     from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
     limit = kwargs.pop("limit", None)
+    reverse = kwargs.pop("reverse", False)
     snapshots = build_snapshot_queryset(**kwargs)
+    snapshots = snapshots.order_by(kwargs.get("sort") or "-created_at", "-id")
+    if reverse:
+        snapshots = snapshots.reverse()
     if kwargs.get("resume"):
-        snapshots = snapshots.filter(timestamp__lte=kwargs["resume"])
+        snapshots = snapshots.filter(**{"timestamp__gte" if reverse else "timestamp__lte": kwargs["resume"]})
     snapshots = snapshots.select_related("crawl")
     if limit is not None and limit > 0:
         snapshot_ids = list(snapshots.values_list("id", flat=True)[:limit])
-        snapshots = Snapshot.objects.filter(id__in=snapshot_ids).select_related("crawl")
+        snapshots = snapshots.filter(id__in=snapshot_ids)
 
     return snapshots
 
@@ -73,6 +77,7 @@ def reindex_snapshots(
     search_plugins: list[str],
     batch_size: int,
     collect_ids: bool = False,
+    reverse: bool = False,
     wait_for_turn=None,
 ) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
@@ -94,16 +99,15 @@ def reindex_snapshots(
             plugin=plugin_name,
             status__in=completed_statuses,
         )
-        candidates = snapshots.annotate(has_completed_index=Exists(completed_result)).filter(has_completed_index=False).order_by("id")
-        after_id = None
+        candidates = snapshots.annotate(has_completed_index=Exists(completed_result)).filter(has_completed_index=False)
+        candidates = candidates.order_by("created_at", "id") if reverse else candidates.order_by("-created_at", "-id")
+        remaining = candidates.select_related(None).only("id", "timestamp", "created_at").paged_iterator(chunk_size=batch_size)
         while True:
             if wait_for_turn:
                 wait_for_turn()
-            page = candidates.filter(id__gt=after_id) if after_id is not None else candidates
-            batch = list(page.select_related(None).only("id", "timestamp")[:batch_size])
+            batch = list(islice(remaining, batch_size))
             if not batch:
                 break
-            after_id = batch[-1].id
             records = [{"type": "Snapshot", "id": str(snapshot.id)} for snapshot in batch]
             stats["processed"] += len(batch)
             stats["requested"] += len(batch)
@@ -161,18 +165,18 @@ def update(
     continuous: bool = False,
     index_only: bool = False,
     migrate_only: bool = False,
+    rescan: bool = False,
+    reverse: bool = False,
     stop_daemon_stack: bool = True,
 ) -> None:
     """
-    Update snapshots: migrate old dirs, reconcile DB, and re-queue for archiving.
+    Update the database without forcing a filesystem scan by default.
 
-    Three-phase operation (without filters):
-    - Phase 1: Drain legacy archive/ directories into the current layout
-    - Phase 2: Select only stale fs_version rows through the indexed column
-    - Phase 3: Run queued snapshot-level filesystem maintenance until idle
-
-    With filters: Only phase 2 (DB query), no filesystem operations.
-    Without filters: All phases (full update).
+    --migrate-only: eagerly finish pending filesystem migrations for known rows.
+    --index-only: reconcile known snapshot assets/metadata and backfill search.
+    --rescan: discover filesystem orphans first, then repair known snapshots.
+    Explicit scans default to newest first; --reverse selects oldest first.
+    Filters select existing database snapshots for targeted updates.
     """
 
     from rich import print
@@ -248,12 +252,28 @@ def update(
     exit_code = 0
 
     try:
+        if rescan and (is_filtered_update or resume or continuous):
+            raise click.UsageError("--rescan cannot be combined with filters, --resume, or --continuous; rerun it to resume automatically.")
+        if rescan:
+            migrate_only = True  # Discovery must not schedule new captures.
         wait_for_turn()
 
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             while True:
-                do_migrate = migrate_only or not index_only
-                do_index = index_only or not migrate_only
+                if rescan:
+                    from archivebox.cli.rescan import rescan_snapshots
+
+                    rescan_snapshots(
+                        wait_for_turn=wait_for_turn,
+                        reverse=reverse,
+                        scan_legacy=lambda: drain_old_archive_dirs(batch_size=batch_size, reverse=reverse),
+                    )
+                do_migrate = migrate_only or is_filtered_update and not index_only
+                do_index = index_only
+                if not do_migrate and not do_index:
+                    print(
+                        "[*] Collection ready. Filesystem migrations remain lazy; use --migrate-only, --index-only, or --rescan for explicit scans.",
+                    )
 
                 if do_migrate:
                     if (
@@ -285,7 +305,8 @@ def update(
                             after=after,
                             resume=resume,
                             batch_size=batch_size,
-                            queue_for_archiving=True,
+                            queue_for_archiving=not migrate_only,
+                            reverse=reverse,
                             wait_for_turn=wait_for_turn,
                         )
                         print_stats(stats)
@@ -293,17 +314,13 @@ def update(
                     else:
                         stats_combined = {"phase1": {}, "phase2": {}}
 
-                        print("[*] Phase 1: Draining old archive/ directories (0.8.x → 0.9.x migration)...")
-                        stats_combined["phase1"] = drain_old_archive_dirs(
-                            resume_from=resume,
-                            batch_size=batch_size,
-                        )
-
                         print("[*] Phase 2: Selecting database snapshots with stale filesystem versions...")
                         stats_combined["phase2"] = process_all_db_snapshots(
                             batch_size=batch_size,
                             resume=resume,
                             wait_for_turn=wait_for_turn,
+                            reverse=reverse,
+                            migrate=True,
                         )
                         print_combined_stats(stats_combined)
                     # The due selectors are indexed and cheap when empty, so
@@ -311,7 +328,7 @@ def update(
                     # whole-table counts merely to decide whether to call it.
                     print("[*] Phase 3: Running filesystem maintenance until idle...")
                     if is_filtered_update:
-                        for snapshot_id in sorted(touched_snapshot_ids):
+                        for snapshot_id in sorted(touched_snapshot_ids, reverse=not reverse):
                             run_scoped_runner("--snapshot-id", snapshot_id)
                     elif migrate_only:
                         run_scoped_runner("--maintenance-only")
@@ -319,6 +336,25 @@ def update(
                         run_scoped_runner()
 
                 if do_index:
+                    from archivebox.cli.rescan import reconcile_known_snapshots
+
+                    snapshots = _build_filtered_snapshots_queryset(
+                        filter_patterns=filter_patterns,
+                        filter_type=filter_type,
+                        status=status,
+                        url__icontains=url__icontains,
+                        url__istartswith=url__istartswith,
+                        tag=tag,
+                        crawl_id=crawl_id,
+                        limit=limit,
+                        sort=sort,
+                        search=search,
+                        before=before,
+                        after=after,
+                        resume=resume,
+                        reverse=reverse,
+                    )
+                    reconcile_known_snapshots(snapshots, wait_for_turn=wait_for_turn)
                     search_plugins = _get_search_indexing_plugins()
                     if not search_plugins:
                         print("[*] No search indexing plugins are available, nothing to backfill.")
@@ -337,12 +373,14 @@ def update(
                             before=before,
                             after=after,
                             resume=resume,
+                            reverse=reverse,
                         )
                         stats = reindex_snapshots(
                             snapshots,
                             search_plugins=search_plugins,
                             batch_size=batch_size,
                             collect_ids=is_filtered_update,
+                            reverse=reverse,
                             wait_for_turn=wait_for_turn,
                         )
                         print_index_stats(stats)
@@ -358,6 +396,10 @@ def update(
         exit_code = 130
         exact_resume = err.__dict__.get("archivebox_resume")
         resume_cmd = ["archivebox", "update"]
+        if rescan:
+            resume_cmd.append("--rescan")
+        if reverse:
+            resume_cmd.append("--reverse")
         if migrate_only:
             resume_cmd.append("--migrate-only")
         if index_only:
@@ -405,7 +447,7 @@ def update(
             stop_own_supervisord_process()
 
 
-def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500) -> dict[str, int]:
+def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500, reverse: bool = False) -> dict[str, int]:
     """
     Drain old archive/ directories (0.8.x → 0.9.x migration).
 
@@ -471,18 +513,18 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
 
     # Scan real legacy directories only; these still contain data to migrate.
     entries = [
-        (e.stat().st_mtime, e.path)
+        (float(e.name), e.path)
         for e in all_entries
         if e.is_dir(follow_symlinks=False) and Snapshot.is_legacy_archive_dir(Path(e.path))  # Skip symlinks and 0.9.x roots
     ]
-    entries.sort(reverse=True)  # Newest first
+    entries.sort(reverse=not reverse)  # Newest first
     print(f"[*] Found {len(entries)} old directories to drain")
 
     for mtime, entry_path in entries:
         entry_path = Path(entry_path)
 
         # Resume from timestamp if specified
-        if resume_from and entry_path.name > resume_from:
+        if resume_from and (entry_path.name < resume_from if reverse else entry_path.name > resume_from):
             continue
 
         stats["processed"] += 1
@@ -584,7 +626,13 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
     return stats
 
 
-def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, wait_for_turn=None) -> dict[str, int]:
+def process_all_db_snapshots(
+    batch_size: int = 500,
+    resume: str | None = None,
+    wait_for_turn=None,
+    reverse: bool = False,
+    migrate: bool = False,
+) -> dict[str, int]:
     """Queue only snapshots whose indexed filesystem version is stale."""
     from archivebox.core.models import Snapshot
     from django.db.models import Q
@@ -601,21 +649,27 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     }
     queryset = Snapshot.objects.filter(fs_version__in=Snapshot._FS_VERSION_MIGRATION_PATHS)
     if resume:
-        queryset = queryset.filter(timestamp__lte=resume)
+        queryset = queryset.filter(**{"timestamp__gte" if reverse else "timestamp__lte": resume})
     initial_now = timezone.now()
-    rows_to_wake = queryset.filter(Q(retry_at__isnull=True) | Q(retry_at__gt=initial_now))
-    after_id = None
+    rows_to_wake = queryset if migrate else queryset.filter(Q(retry_at__isnull=True) | Q(retry_at__gt=initial_now))
+    ordered = rows_to_wake.order_by("created_at", "id") if reverse else rows_to_wake.order_by("-created_at", "-id")
+    remaining = ordered.select_related("crawl__created_by").paged_iterator(chunk_size=batch_size)
     while True:
         if wait_for_turn:
             wait_for_turn()
-        page = rows_to_wake.filter(id__gt=after_id) if after_id is not None else rows_to_wake
-        batch = list(page.only("id", "fs_version", "modified_at").order_by("id")[:batch_size])
+        batch = list(islice(remaining, batch_size))
         if not batch:
             break
-        after_id = batch[-1].id
         now = timezone.now()
         updated = 0
         for snapshot in batch:
+            if migrate:
+                from archivebox.services.runner import run_snapshot_maintenance
+
+                if wait_for_turn:
+                    wait_for_turn()
+                run_snapshot_maintenance(str(snapshot.id))
+                continue
             updated += int(
                 snapshot.safe_update(
                     {"retry_at": now, "modified_at": now},
@@ -647,6 +701,7 @@ def process_filtered_snapshots(
     resume: str | None,
     batch_size: int,
     queue_for_archiving: bool = True,
+    reverse: bool = False,
     wait_for_turn=None,
 ) -> dict[str, Any]:
     """Process snapshots matching filters (DB query only)."""
@@ -669,6 +724,7 @@ def process_filtered_snapshots(
         before=before,
         after=after,
         resume=resume,
+        reverse=reverse,
     )
 
     total = snapshots.count()
@@ -687,6 +743,10 @@ def process_filtered_snapshots(
             stats["snapshot_ids"].append(str(snapshot.id))
             update_values = {}
             updated = 0
+            if not queue_for_archiving and snapshot.fs_migration_needed:
+                from archivebox.services.runner import run_snapshot_maintenance
+
+                run_snapshot_maintenance(str(snapshot.id))
             if queue_for_archiving:
                 update_values.update(
                     {
@@ -784,6 +844,12 @@ def print_index_stats(stats: dict[str, Any]) -> None:
 @click.option("--continuous", is_flag=True, help="Run continuously as background worker")
 @click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content")
 @click.option("--migrate-only", is_flag=True, help="Only migrate filesystem and update database/index state")
+@click.option("--reverse", is_flag=True, help="Process oldest first instead of newest first for explicit scans")
+@click.option(
+    "--rescan",
+    is_flag=True,
+    help="Scan all snapshot directories for orphans and metadata repairs (slow, opt-in, safe to restart)",
+)
 @snapshot_filter_options(default_filter_type="exact")
 @docstring(update.__doc__)
 def main(**kwargs):

archivebox/cli/archivebox_version.py

@@ -80,20 +80,6 @@ def _render_binary_abspath(abspath: str):
     return Text(abspath, style="green")
 
 
-def _binary_record_matches_runtime(installed, lib_dir: Path) -> bool:
-    if not installed or not installed.is_valid or not installed.version:
-        return False
-    try:
-        abspath = Path(installed.abspath).expanduser().resolve(strict=False)
-        if not abspath.exists():
-            return False
-        if installed.binprovider not in {"", "env", "apt", "brew"}:
-            abspath.relative_to(lib_dir)
-    except (OSError, ValueError):
-        return False
-    return True
-
-
 def _binary_row_dedupe_key(
     *,
     display_name: str,
@@ -127,7 +113,6 @@ def version(
 
     from rich.panel import Panel
     from rich.console import Console
-    from rich.live import Live
     from abx_dl.tables import binary_dependency_status, binary_dependency_table
 
     from archivebox.config import CONSTANTS
@@ -263,7 +248,6 @@ def version(
     prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
     failures = []
     seen_failures: set[str] = set()
-    seen_rows: set[tuple[str, str, str, str]] = set()
 
     from archivebox.plugins.discovery import get_enabled_plugins, get_plugin_catalog
     from abx_dl.config import get_required_binary_requests
@@ -272,37 +256,29 @@ def version(
     from abxpkg.binary_service import BinaryEvent, BinaryService
 
     plugins = get_plugin_catalog()
-    enabled_plugins = plugins.select(get_enabled_plugins(config=config))
-    enabled_plugin_names = set(enabled_plugins)
-    runtime_config = normalize_runtime_config(config.for_crawl(), json_safe=False)
+    enabled_plugin_names = set(get_enabled_plugins(config=config))
+    runtime_config = normalize_runtime_config(dict(config.items()), json_safe=False)
     derived_config: dict[str, object] = {}
-    db_binaries = {}
-    db_available = False
     if in_data_dir:
         try:
             from archivebox.config.django import setup_django
 
             setup_django()
 
-            from archivebox.machine.models import Machine, Binary
+            from archivebox.machine.models import Machine
 
             machine = Machine.current()
             derived_config = normalize_runtime_config(machine.config, json_safe=False)
-            for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
-                if _binary_record_matches_runtime(binary, config.ABXPKG_LIB_DIR):
-                    db_binaries.setdefault(binary.name, binary)
-            db_available = True
 
         except Exception as e:
             prnt()
-            prnt("", f"[yellow]Warning: Could not query collection binary records; resolving through abxpkg: {e}[/yellow]")
+            prnt("", f"[yellow]Warning: Could not query collection machine config; resolving through abxpkg: {e}[/yellow]")
 
     declared_binary_specs: dict[str, dict[str, object]] = {}
     for plugin_name, plugin in plugins.items():
         if not plugin_may_have_requested_binary(plugin):
             continue
-        plugin_requested = bool(requested_names)
-        plugin_enabled = plugin_name in enabled_plugin_names
+        plugin_requested = plugin_name.lower() in {name.lower() for name in requested_names}
         binary_records = get_required_binary_requests(
             plugin,
             plugin.config.required_binaries,
@@ -320,10 +296,9 @@ def version(
             display_name = logical_name
             if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
                 continue
-            if not plugin_enabled and not requested_names:
-                continue
-            if _binary_record_matches_runtime(db_binaries.get(logical_name), config.ABXPKG_LIB_DIR):
-                continue
+            # This is a diagnostic: recheck the provider instead of trusting an
+            # installation record or a cached version, including disabled plugins.
+            binary_record["no_cache"] = True
             signature = json.dumps(binary_record, sort_keys=True, default=str)
             declared_binary_specs.setdefault(signature, binary_record)
 
@@ -341,126 +316,47 @@ def version(
 
         loaded_binaries = asyncio.run(resolve_declared_binaries())
 
-    rows: list[dict[str, object]] = []
+    rows_by_binary: dict[str, dict] = {}
     any_rows = False
     any_available = False
     compact_paths = console.is_terminal
-    live_enabled = console.is_terminal
-    live_cm = Live(binary_dependency_table(rows), console=console, refresh_per_second=8) if live_enabled else None
-
-    def emit_row(row: dict[str, object]) -> None:
-        rows.append(row)
-        if live_cm is not None:
-            live_cm.update(binary_dependency_table(rows), refresh=True)
-
-    if live_cm is not None:
-        live_cm.start()
-    try:
-        for plugin_name, plugin in plugins.items():
-            if not plugin_may_have_requested_binary(plugin):
-                continue
-            plugin_requested = bool(requested_names)
-            plugin_enabled = plugin_name in enabled_plugin_names
-            binary_records = get_required_binary_requests(
-                plugin,
-                plugin.config.required_binaries,
-                overrides=runtime_config,
-                derived_overrides=derived_config,
-                run_output_dir=CONSTANTS.DATA_DIR,
+    for plugin_name, plugin in plugins.items():
+        if not plugin_may_have_requested_binary(plugin):
+            continue
+        plugin_requested = plugin_name.lower() in {name.lower() for name in requested_names}
+        plugin_enabled = plugin_name in enabled_plugin_names
+        disabled_by = ", ".join(
+            f"{dependency.enabled_key}=False"
+            for dependency in plugins.select([plugin_name]).values()
+            if not config.get(dependency.enabled_key, True)
+        )
+        binary_records = get_required_binary_requests(
+            plugin,
+            plugin.config.required_binaries,
+            overrides=runtime_config,
+            derived_overrides=derived_config,
+            run_output_dir=CONSTANTS.DATA_DIR,
+        )
+        for binary_record in binary_records:
+            binary_record["no_cache"] = True
+            actual_name = str(binary_record["name"])
+            logical_name = (
+                Path(actual_name).expanduser().name
+                if ("/" in actual_name or "\\" in actual_name or actual_name.startswith("~"))
+                else actual_name
             )
-            for binary_record in binary_records:
-                actual_name = str(binary_record["name"])
-                logical_name = (
-                    Path(actual_name).expanduser().name
-                    if ("/" in actual_name or "\\" in actual_name or actual_name.startswith("~"))
-                    else actual_name
-                )
-                display_name = logical_name
-                if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
-                    continue
+            display_name = logical_name
+            if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
+                continue
 
-                installed = db_binaries.get(logical_name) if db_available else None
-                if _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
-                    abspath = installed.abspath
-                    version_str = (installed.version or "unknown")[:15]
-                    provider = (installed.binprovider or "env")[:8]
-                    valid = True
-                elif not plugin_enabled and not requested_names:
-                    # `archivebox version` is expected to verify the active runtime, not
-                    # cold-load every optional plugin provider. Migration and status
-                    # checks often run with PLUGINS narrowed to a tiny set; resolving
-                    # disabled plugin binaries there can spend most of the command on
-                    # providers the current collection will never execute.
-                    continue
-                else:
-                    loaded = loaded_binaries[json.dumps(binary_record, sort_keys=True, default=str)]
-                    abspath = loaded.abspath if loaded is not None else ""
-                    version_str = str(loaded.version or "unknown")[:15] if loaded is not None else "unknown"
-                    provider = str(loaded.binprovider or "env")[:8] if loaded is not None else "env"
-                    valid = loaded is not None
-
-                any_rows = True
-                if valid:
-                    display_path = (
-                        _format_binary_abspath(
-                            abspath,
-                            pwd=Path.cwd(),
-                            lib_dir=config.ABXPKG_LIB_DIR,
-                            personas_dir=CONSTANTS.PERSONAS_DIR,
-                            home=Path.home(),
-                        )
-                        if compact_paths
-                        else abspath
-                    )
-                    rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
-                    any_available = True
-                else:
-                    rendered_path = "[grey53]not installed[/grey53]"
-                    if plugin_enabled and display_name not in seen_failures:
-                        failures.append(display_name)
-                        seen_failures.add(display_name)
-
-                row_key = _binary_row_dedupe_key(
-                    display_name=display_name,
-                    valid=valid,
-                    version=version_str if valid else "-",
-                    provider=provider if valid else "-",
-                    abspath=abspath,
-                )
-                if row_key in seen_rows:
-                    continue
-                seen_rows.add(row_key)
-
-                emit_row(
-                    {
-                        "plugin": plugin_name,
-                        "status": binary_dependency_status(enabled=plugin_enabled, valid=valid),
-                        "binary": display_name,
-                        "version": version_str if valid else "-",
-                        "provider": provider if valid else "-",
-                        "path": rendered_path,
-                        "style": "" if plugin_enabled else "dim",
-                    },
-                )
-
-        if db_available:
-            for binary_name, installed in db_binaries.items():
-                if requested_names and binary_name not in requested_names:
-                    continue
-                abspath = installed.abspath
-                version_str = (installed.version or "unknown")[:15]
-                provider = (installed.binprovider or "env")[:8]
-                row_key = _binary_row_dedupe_key(
-                    display_name=binary_name,
-                    valid=True,
-                    version=version_str,
-                    provider=provider,
-                    abspath=abspath,
-                )
-                if row_key in seen_rows:
-                    continue
-                seen_rows.add(row_key)
+            loaded = loaded_binaries[json.dumps(binary_record, sort_keys=True, default=str)]
+            abspath = loaded.abspath if loaded is not None else ""
+            version_str = str(loaded.version or "unknown") if loaded is not None else "unknown"
+            provider = str(loaded.binprovider or "env") if loaded is not None else "env"
+            valid = loaded is not None and Path(abspath).is_file()
 
+            any_rows = True
+            if valid:
                 display_path = (
                     _format_binary_abspath(
                         abspath,
@@ -472,24 +368,67 @@ def version(
                     if compact_paths
                     else abspath
                 )
-                emit_row(
-                    {
-                        "plugin": "(database)",
-                        "status": binary_dependency_status(enabled=True, valid=True),
-                        "binary": binary_name,
-                        "version": version_str,
-                        "provider": provider,
-                        "path": _render_binary_abspath(display_path) if compact_paths else display_path,
-                        "style": "dim",
-                    },
-                )
-                any_rows = True
+                rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
                 any_available = True
-    finally:
-        if live_cm is not None:
-            live_cm.stop()
-        else:
-            prnt(binary_dependency_table(rows))
+            else:
+                rendered_path = "[grey53]not installed[/grey53]"
+                if plugin_enabled and display_name not in seen_failures:
+                    failures.append(display_name)
+                    seen_failures.add(display_name)
+
+            if not plugin_enabled:
+                rendered_path = f"[grey53]disabled by {disabled_by or 'PLUGINS configuration'}[/grey53]"
+
+            row = {
+                "plugin": plugin_name,
+                "status": binary_dependency_status(enabled=plugin_enabled, valid=valid),
+                "binary": display_name,
+                "version": version_str if valid else "-",
+                "provider": provider if valid else "-",
+                "path": rendered_path,
+                "style": "" if plugin_enabled else "dim",
+                "enabled": plugin_enabled,
+                "valid": valid,
+                "plugins": [plugin_name],
+                "disabled_by": [disabled_by or "PLUGINS configuration"] if not plugin_enabled else [],
+            }
+            existing = rows_by_binary.get(display_name)
+            if existing is None:
+                rows_by_binary[display_name] = row
+            else:
+                consumers = list(dict.fromkeys([*existing["plugins"], plugin_name]))
+                disabled_reasons = list(dict.fromkeys([*existing["disabled_by"], *row["disabled_by"]]))
+                # Disabled consumers cannot mask an active requirement. If any
+                # active request fails (e.g. a version floor), keep that failure.
+                if plugin_enabled and (not existing["enabled"] or not valid):
+                    existing.update(row)
+                existing["plugins"] = consumers
+                existing["disabled_by"] = disabled_reasons
+                existing["plugin"] = ", ".join(consumers)
+                if not existing["enabled"]:
+                    existing["path"] = f"[grey53]disabled by {', '.join(disabled_reasons)}[/grey53]"
+
+    rows = list(rows_by_binary.values())
+
+    if console.is_terminal and console.width < 120:
+        from rich.text import Text
+
+        for row in rows:
+            heading = Text.from_markup(str(row["status"]))
+            heading.append(f" {row['binary']} {row['version']} ({row['provider']}) · {row['plugin']}")
+            prnt(heading, style=str(row["style"]))
+            path = row["path"]
+            prnt(Text.assemble("   ", path if isinstance(path, Text) else Text.from_markup(str(path))), style=str(row["style"]))
+    else:
+        table = binary_dependency_table(rows)
+        table.expand = False
+        table.title = None
+        table.box = None
+        for column in table.columns:
+            column.width = None
+            column.max_width = None
+            column.ratio = None
+        prnt(table)
 
     if not any_rows:
         prnt("", "[grey53]No required binaries declared for discovered plugins.[/grey53]")

archivebox/cli/rescan.py

@@ -0,0 +1,222 @@
+"""Explicit discovery of portable snapshots; no checkpoint or default tree walk."""
+
+import json
+import os
+import uuid
+from itertools import groupby
+from pathlib import Path
+
+from django.contrib.auth import get_user_model
+from django.core.exceptions import ValidationError
+from django.db import IntegrityError
+from rich import print
+
+from archivebox.config import CONSTANTS
+from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.crawls.models import Crawl
+from archivebox.misc.util import parse_date
+
+
+def _directories(parent: Path):
+    # Names encode dates / UUIDv7 creation time. Newest first within each
+    # subtree, with bounded memory and no traversal into plugin payloads.
+    with os.scandir(parent) as entries:
+        paths = [Path(entry.path) for entry in entries if entry.is_dir(follow_symlinks=False)]
+    yield from sorted(paths, key=lambda path: path.name, reverse=True)
+
+
+def _snapshot_directories(reverse=False):
+    dates = []
+    for user in _directories(CONSTANTS.USERS_DIR):
+        root = user / CONSTANTS.SNAPSHOTS_DIR_NAME
+        if root.is_symlink() or not root.is_dir():
+            continue
+        dates.extend(_directories(root))
+    dates.sort(key=lambda path: path.name, reverse=not reverse)
+    for _, date_group in groupby(dates, key=lambda path: path.name):
+        directories = [directory for date in date_group for domain in _directories(date) for directory in _directories(domain)]
+        yield from sorted(directories, key=lambda path: path.name, reverse=not reverse)
+
+
+def _fields(record, names):
+    return {name: record[name] for name in names.split() if name in record}
+
+
+def _dates(record):
+    return {name: parse_date(record[name]) for name in ("created_at", "modified_at") if record.get(name)}
+
+
+def _restore_dates(obj, record):
+    dates = _dates(record)
+    if dates:
+        type(obj).objects.filter(pk=obj.pk).update(**dates)
+        for name, value in dates.items():
+            setattr(obj, name, value)
+
+
+def _reconcile_directory(directory: Path) -> str:
+    snapshot_id = uuid.UUID(directory.name)
+    snapshot = Snapshot.objects.select_related("crawl__created_by").filter(pk=snapshot_id).first()
+    index = directory / CONSTANTS.JSONL_INDEX_FILENAME
+    if index.is_symlink():
+        raise ValueError("metadata must not be a symlink")
+    if not index.exists():
+        if snapshot is None or Path(snapshot.output_dir) != directory or snapshot.status != "sealed":
+            raise ValueError("missing index.jsonl; cannot reconstruct an unknown or active snapshot")
+        snapshot.write_index_jsonl(output_dir=directory)
+        snapshot.reconcile_filesystem_links()
+        return "Repaired"
+
+    records = [json.loads(line) for line in index.read_text().splitlines() if line.strip()]
+    if any(not isinstance(record, dict) for record in records):
+        raise ValueError("metadata records must be JSON objects")
+    snapshots = [record for record in records if record.get("type") == "Snapshot"]
+    if len(snapshots) != 1:
+        raise ValueError("expected exactly one Snapshot record")
+    record = snapshots[0]
+    if uuid.UUID(record["id"]) != snapshot_id:
+        raise ValueError("snapshot ID conflicts with directory name")
+    if record.get("status") != "sealed":
+        raise ValueError("snapshot is not sealed; finish the source capture first")
+    if record.get("fs_version") not in {Snapshot._fs_current_version(), *Snapshot._FS_VERSION_MIGRATION_PATHS}:
+        raise ValueError("unsupported filesystem version")
+    crawl_id = uuid.UUID(record["crawl_id"])
+    owner = directory.parents[3].name
+    crawl_records = [item for item in records if item.get("type") == "Crawl" and uuid.UUID(item["id"]) == crawl_id]
+    crawl_record = crawl_records[0] if crawl_records else {}
+    if crawl_record.get("created_by", owner) != owner:
+        raise ValueError("crawl owner conflicts with directory path")
+    crawl = Crawl.objects.select_related("created_by").filter(pk=crawl_id).first()
+    imported_crawl = crawl is None
+    if crawl and crawl.created_by.username != owner:
+        raise ValueError("existing crawl ID belongs to a different owner")
+    if snapshot and (snapshot.url != record["url"] or snapshot.crawl_id != crawl_id or snapshot.timestamp != record["timestamp"]):
+        raise ValueError("existing snapshot ID has conflicting identity; refusing to overwrite")
+    if snapshot and Path(snapshot.output_dir) != directory:
+        raise ValueError("existing snapshot has a different storage path")
+
+    # Validate identities before inserting anything. Never turn an ID collision
+    # into a new ID, and never overwrite a newer destination row.
+    # write_index_jsonl retains historical hook records before appending the
+    # canonical DB rows. The final record for each hook is authoritative.
+    results = {(item.get("plugin"), item.get("hook_name", "")): item for item in records if item.get("type") == "ArchiveResult"}.values()
+    pending = []
+    for result in results:
+        result_id = uuid.UUID(result["id"])
+        if uuid.UUID(result["snapshot_id"]) != snapshot_id:
+            raise ValueError("ArchiveResult belongs to another snapshot")
+        existing = ArchiveResult.objects.filter(pk=result_id).first()
+        identity = (snapshot_id, result["plugin"], result.get("hook_name", ""))
+        if existing:
+            if (existing.snapshot_id, existing.plugin, existing.hook_name) != identity:
+                raise ValueError("existing ArchiveResult ID has conflicting identity")
+        elif ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin=identity[1], hook_name=identity[2]).exists():
+            raise ValueError("existing hook has a different ArchiveResult ID")
+        else:
+            pending.append(result)
+
+    imported = snapshot is None
+    if imported:
+        if Snapshot.objects.filter(timestamp=record["timestamp"]).exists():
+            raise ValueError("timestamp belongs to another snapshot")
+        if Snapshot.objects.filter(crawl_id=crawl_id, url=record["url"]).exists():
+            raise ValueError("crawl URL belongs to another snapshot")
+        user, _ = get_user_model().objects.get_or_create(
+            username=owner,
+            defaults={"is_active": False, "password": "!"},
+        )
+        if crawl is None:
+            # Old exports contain only crawl_id. Keep that identity but report
+            # that absent crawl metadata cannot be reconstructed exactly.
+            if not crawl_record:
+                print(f"[yellow]    {directory}: no Crawl record; preserving ID with recovered metadata[/yellow]")
+            crawl = Crawl(
+                id=crawl_id,
+                created_by=user,
+                status="sealed",
+                retry_at=None,
+                **_fields(crawl_record, "urls max_depth config tags_str label"),
+            )
+            crawl.save(force_insert=True)
+            _restore_dates(crawl, crawl_record)
+        snapshot = Snapshot.create_from_directory(directory, crawl=crawl, preserve_identity=True)
+        if snapshot is None:
+            raise ValueError("invalid snapshot metadata")
+        snapshot.status = "sealed"
+        snapshot.retry_at = None
+        if "config" not in record:
+            snapshot.config = {"PERMISSIONS": "private"}
+        if Path(snapshot.get_storage_path_for_version(Snapshot._fs_current_version())) != directory:
+            raise ValueError("metadata does not match its owner/date/domain directory")
+        snapshot.save(force_insert=True)
+        _restore_dates(snapshot, record)
+
+    # Each insert is durable. An interruption after the Snapshot insert is not
+    # completion: the next pass still imports missing result IDs and tags.
+    for result in pending:
+        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=True)
+    tags_before = set(snapshot.tags.values_list("name", flat=True))
+    snapshot._merge_tags_from_index(record)
+    tags_changed = tags_before != set(snapshot.tags.values_list("name", flat=True))
+    link = Path(snapshot.crawl.output_dir) / "snapshots" / snapshot.extract_domain_from_url(snapshot.url) / str(snapshot.id)
+    links_ok = link.is_symlink() and link.resolve() == directory.resolve()
+    if not links_ok:
+        snapshot.reconcile_filesystem_links()
+    if snapshot.fs_migration_needed:
+        snapshot.migrate_filesystem_to_current_version(source_dir=directory)
+    if imported:
+        _restore_dates(snapshot, record)
+    if imported_crawl:
+        _restore_dates(crawl, crawl_record)
+    # Leave source metadata untouched, including process/binary records that
+    # are historical evidence rather than runnable destination process state.
+    return "Imported" if imported else "Repaired" if pending or tags_changed or not links_ok else "Unchanged"
+
+
+def reconcile_known_snapshots(snapshots, wait_for_turn=None):
+    """Explicit metadata / asset repair; normal updates never call this scan."""
+    for snapshot in snapshots.filter(status="sealed").paged_iterator():
+        if wait_for_turn:
+            wait_for_turn()
+        if snapshot.fs_migration_needed:
+            snapshot.migrate_filesystem_to_current_version()
+        directory = Path(snapshot.output_dir)
+        if not directory.is_dir():
+            raise ValueError(f"Missing snapshot directory: {directory}")
+        _reconcile_directory(directory)
+        for result in snapshot.archiveresult_set.all():
+            result.update_output_metadata_from_filesystem(snapshot_dir=directory, full_scan=True)
+
+
+def rescan_snapshots(wait_for_turn=None, reverse=False, scan_legacy=None):
+    stats = dict.fromkeys(("Imported", "Repaired", "Unchanged", "Errors"), 0)
+    print("[*] Rescanning snapshot directories (O(N), orphans first; completed records are reused)...")
+    known = {str(pk) for pk in Snapshot.objects.values_list("pk", flat=True)}
+    # Two streaming passes avoid enumerating the entire archive before the
+    # first import. Rows imported before interruption move to the second pass
+    # next time; only one day's directory names are held in memory at a time.
+    for existing in (False, True):
+        if existing and scan_legacy:
+            scan_legacy()
+        for directory in _snapshot_directories(reverse=reverse):
+            if (directory.name.replace("-", "") in known) != existing:
+                continue
+            if wait_for_turn:
+                wait_for_turn()
+            try:
+                outcome = _reconcile_directory(directory)
+                if directory.name.replace("-", "") in known:
+                    for result in ArchiveResult.objects.filter(snapshot_id=uuid.UUID(directory.name)):
+                        result.update_output_metadata_from_filesystem(snapshot_dir=directory, full_scan=True)
+                stats[outcome] += 1
+                if outcome != "Unchanged":
+                    print(f"    {outcome}: {directory}")
+            except (OSError, ValueError, KeyError, TypeError, ValidationError, IntegrityError) as err:
+                stats["Errors"] += 1
+                print(f"[yellow]    Unresolved: {directory}: {err}[/yellow]")
+            if sum(stats.values()) % 100 == 0:
+                print("    " + ", ".join(f"{key}: {value}" for key, value in stats.items()))
+    print("    " + ", ".join(f"{key}: {value}" for key, value in stats.items()))
+    if stats["Errors"]:
+        raise SystemExit(1)
+    return stats

archivebox/config/common.py

@@ -83,12 +83,16 @@ def base_url_from_legacy_server_config(raw_config: Mapping[str, object]) -> str:
             return candidate
 
     listen_host = str(raw_config.get("LISTEN_HOST") or "").strip()
-    if listen_host and listen_host.lower() != "archivebox.localhost:8000":
+    if listen_host and listen_host.lower() not in {"archivebox.localhost:8000", "archivebox.localhost:5797"}:
         return normalize(listen_host)
 
     csrf_origins = [normalize(origin) for origin in str(raw_config.get("CSRF_TRUSTED_ORIGINS") or "").split(",")]
     csrf_origins = [origin for origin in csrf_origins if origin]
-    nondefault_csrf_origins = [origin for origin in csrf_origins if origin != "http://admin.archivebox.localhost:8000"]
+    nondefault_csrf_origins = [
+        origin
+        for origin in csrf_origins
+        if origin not in {"http://admin.archivebox.localhost:8000", "http://admin.archivebox.localhost:5797"}
+    ]
     if len(nondefault_csrf_origins) == 1:
         return nondefault_csrf_origins[0]
 
@@ -314,7 +318,7 @@ class ServerConfig(BaseConfigSet):
     )
 
     SECRET_KEY: str = Field(default_factory=lambda: "".join(secrets.choice("abcdefghijklmnopqrstuvwxyz0123456789_") for _ in range(50)))
-    BIND_ADDR: str = Field(default="127.0.0.1:8000")
+    BIND_ADDR: str = Field(default="127.0.0.1:5797")
     BASE_URL: str = Field(default="")
     ALLOWED_HOSTS: str = Field(default="*")
     CSRF_TRUSTED_ORIGINS: str = Field(default="")

archivebox/core/admin_snapshots.py

@@ -5,9 +5,10 @@ from functools import lru_cache
 from types import SimpleNamespace
 
 from django.contrib import admin, messages
+from django.core.exceptions import PermissionDenied
 from django.urls import path, reverse
 from django.shortcuts import get_object_or_404, redirect
-from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed
+from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed, Http404
 from django.utils import timezone
 from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
@@ -20,6 +21,7 @@ from archivebox.config.common import get_config
 from archivebox.misc.util import htmldecode, urldecode
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.misc.logging_util import printable_filesize
+from archivebox.misc.serve_static import serve_static_with_byterange_support
 from archivebox.search.admin import SearchResultsAdminMixin, SearchResultsChangeList
 from archivebox.search.views import admin_snapshot_search_stream_view
 from archivebox.core.routes_util import build_snapshot_url, build_web_url
@@ -32,7 +34,7 @@ from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 from archivebox.core.models import Tag, Snapshot, ArchiveResult
 from archivebox.crawls.models import Crawl
 from archivebox.core.admin_archiveresults import render_archiveresults_list
-from archivebox.core.preview_util import EXTENSION_SCREENSHOT_PLUGIN
+from archivebox.core.preview_util import PREVIEW_PLUGINS, snapshot_preview_candidates, render_snapshot_preview
 from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
@@ -46,6 +48,12 @@ from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget
 GLOBAL_CONTEXT = {}
 
 SNAPSHOT_PERMISSION_META = PERMISSIONS_META
+GRID_PREVIEW_OUTPUTS = {
+    ("screenshot", "screenshot.png"),
+    ("chrome_extension_screenshot", "screenshot-1.png"),
+    ("chrome_extension_screenshot", "screenshot.png"),
+    ("favicon", "favicon.ico"),
+}
 
 
 @lru_cache(maxsize=1)
@@ -272,7 +280,17 @@ class SnapshotChangeList(SearchResultsChangeList):
                 SimpleNamespace(plugin=plugin, status=status, output_size=output_size, output_files=output_files),
             )
 
+        response_results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
+        for result in ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, plugin__in=PREVIEW_PLUGINS).only(
+            "snapshot_id",
+            "plugin",
+            "status",
+            "output_files",
+        ):
+            response_results_by_snapshot[result.snapshot_id].append(result)
+
         for obj in self.result_list:
+            obj._preview_results = response_results_by_snapshot[obj.pk]
             counts = status_counts_by_snapshot.get(obj.pk, {})
             total = int(counts.get("total") or 0)
             succeeded = int(counts.get("succeeded") or 0)
@@ -569,6 +587,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         custom_urls = [
             path("grid/", self.admin_site.admin_view(self.grid_view), name="grid"),
             path("search-stream/", self.admin_site.admin_view(self.search_stream_view), name="core_snapshot_search_stream"),
+            path(
+                "<path:object_id>/preview/<str:plugin>/<path:filename>",
+                self.admin_site.admin_view(self.preview_view, cacheable=True),
+                name="core_snapshot_preview",
+            ),
             path("<path:object_id>/redo-failed/", self.admin_site.admin_view(self.redo_failed_view), name="core_snapshot_redo_failed"),
             path(
                 "<path:object_id>/set-permissions/",
@@ -578,6 +601,32 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         ]
         return custom_urls + urls
 
+    def preview_view(self, request, object_id, plugin, filename):
+        """Serve grid thumbnails from the authenticated admin origin."""
+        if plugin not in PREVIEW_PLUGINS:
+            raise Http404("Unsupported snapshot preview")
+        snapshot = get_object_or_404(Snapshot, pk=object_id)
+        if not self.has_view_or_change_permission(request, snapshot):
+            raise PermissionDenied
+        candidate = next(
+            (
+                candidate
+                for candidate in snapshot_preview_candidates(snapshot)
+                if candidate["plugin"] == plugin and candidate["filename"] == filename
+            ),
+            None,
+        )
+        if candidate is None:
+            raise Http404("Snapshot preview does not exist")
+        output_path = candidate["path"]
+        request.archivebox_cache_policy = "private"
+        response = serve_static_with_byterange_support(request, output_path, document_root=snapshot.output_dir)
+        if response.status_code != 304 and not response.headers.get("Content-Type", "").lower().startswith("image/"):
+            raise Http404("Snapshot preview is not an image")
+        response.headers["Vary"] = "Cookie"
+        response.headers["X-Content-Type-Options"] = "nosniff"
+        return response
+
     def search_stream_view(self, request):
         return admin_snapshot_search_stream_view(self, request)
 
@@ -938,79 +987,31 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return mark_safe(f'<span class="tags-inline-editor">{tags_html}</span>')
 
     def _get_preview_data(self, obj):
-        request = self.request
-        config = request.archivebox_config
-        results = self._get_prefetched_results(obj)
-        if results is not None:
-            results = [r for r in results if r.plugin in ("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")]
-        else:
-            results = list(
-                obj.archiveresult_set.filter(plugin__in=("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")).only(
-                    "snapshot_id",
-                    "plugin",
-                    "status",
-                    "output_files",
-                    "output_str",
-                ),
-            )
-
-        def result_output_path(result, filename: str) -> str | None:
-            output_files = result.output_files or {}
-            file_info = output_files.get(filename)
-            if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
-                return None
-            if file_info.get("root_relative"):
-                return filename
-            return f"{result.plugin}/{filename}"
-
-        def result_urls(plugin: str, filenames: tuple[str, ...]) -> list[str]:
-            urls: list[str] = []
-            for result in results:
-                if result.plugin != plugin or result.status != ArchiveResult.StatusChoices.SUCCEEDED:
-                    continue
-                for filename in filenames:
-                    output_path = result_output_path(result, filename)
-                    if output_path:
-                        urls.append(build_snapshot_url(str(obj.id), output_path, request=request, config=config))
-            return urls
-
-        screenshot_urls = result_urls("screenshot", ("screenshot.png",))
-        extension_screenshot_urls = result_urls(EXTENSION_SCREENSHOT_PLUGIN, ("screenshot-1.png", "screenshot.png"))
-        favicon_urls = result_urls("favicon", ("favicon.ico",))
-
-        if not screenshot_urls and not extension_screenshot_urls and not favicon_urls:
+        candidates = snapshot_preview_candidates(obj)
+        if not candidates:
             return None
-
-        all_screenshot_urls = [*screenshot_urls, *extension_screenshot_urls]
-        if all_screenshot_urls:
-            img_url = all_screenshot_urls[0]
-            fallbacks = [*all_screenshot_urls[1:], *favicon_urls]
-            img_alt = "Screenshot"
-            preview_class = "screenshot"
-        else:
-            img_url = favicon_urls[0]
-            fallbacks = favicon_urls[1:]
-            img_alt = "Favicon"
-            preview_class = "favicon"
-
-        fallback_list = ",".join(fallbacks)
-        onerror_js = (
-            "this.dataset.fallbacks && this.dataset.fallbacks.length ? "
-            "(this.src=this.dataset.fallbacks.split(',').shift(), "
-            "this.dataset.fallbacks=this.dataset.fallbacks.split(',').slice(1).join(',')) : "
-            "this.remove()"
-        )
-
+        urls = [
+            build_snapshot_url(str(obj.id), candidate["path"], request=self.request, config=self.request.archivebox_config)
+            for candidate in candidates
+        ]
+        favicons = [url for candidate, url in zip(candidates, urls) if candidate["kind"] == "favicon"]
         return {
-            "img_url": img_url,
-            "img_alt": img_alt,
-            "preview_class": preview_class,
-            "onerror_js": onerror_js,
-            "fallback_list": fallback_list,
-            "favicon_url": favicon_urls[0] if favicon_urls else "",
-            "favicon_fallback_list": ",".join(favicon_urls[1:]),
+            "img_url": urls[0],
+            "fallback_list": ",".join(urls[1:]),
+            "img_alt": "Favicon" if candidates[0]["kind"] == "favicon" else "Screenshot",
+            "preview_class": "favicon" if candidates[0]["kind"] == "favicon" else "screenshot",
+            "favicon_url": favicons[0] if favicons else "",
+            "favicon_fallback_list": ",".join(favicons[1:]),
         }
 
+    def _render_preview(self, obj, width="100px", height="100px"):
+        return render_snapshot_preview(
+            obj,
+            lambda candidate: reverse("admin:core_snapshot_preview", args=(obj.pk, candidate["plugin"], candidate["filename"])),
+            width=width,
+            height=height,
+        )
+
     @admin.display(description="", empty_value="")
     def url_favicon(self, obj):
         preview = self._get_preview_data(obj)
@@ -1041,42 +1042,24 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Preview", empty_value="")
     def preview_icon(self, obj):
-        preview = self._get_preview_data(obj)
-        if not preview:
-            return None
-
         return format_html(
-            '<img src="{}" alt="{}" class="snapshot-preview {}" decoding="async" loading="lazy" onerror="{}" data-fallbacks="{}">',
-            preview["img_url"],
-            preview["img_alt"],
-            preview["preview_class"],
-            preview["onerror_js"],
-            preview["fallback_list"],
+            '<a href="{}" title="Open snapshot details">{}</a>',
+            build_web_url(f"/{obj.archive_path_from_db}/index.html", request=self.request, config=self.request.archivebox_config),
+            self._render_preview(obj),
         )
 
     @admin.display(description=" ", empty_value="")
     def snapshot_summary(self, obj):
         request = self.request
         config = request.archivebox_config
-        preview = self._get_preview_data(obj)
         stats = self._get_progress_stats(obj)
         archive_size = stats["output_size"] or 0
         size_txt = printable_filesize(archive_size) if archive_size else "pending"
-        screenshot_html = ""
-
-        if preview:
-            screenshot_html = format_html(
-                '<a href="{href}" title="Open snapshot live view" style="display:block; flex:0 0 220px; width:220px;">'
-                '<img src="{src}" alt="{alt}" decoding="async" loading="lazy" onerror="{onerror}" data-fallbacks="{fallbacks}" '
-                'style="display:block; width:100%; max-width:220px; aspect-ratio: 16 / 10; object-fit: cover; object-position: top; '
-                'border-radius: 10px; border: 1px solid #e2e8f0; background: #f8fafc;">'
-                "</a>",
-                href=build_web_url(f"/{obj.archive_path}", request=request, config=config),
-                src=preview["img_url"],
-                alt=preview["img_alt"],
-                onerror=preview["onerror_js"],
-                fallbacks=preview["fallback_list"],
-            )
+        screenshot_html = format_html(
+            '<a href="{}" title="Open snapshot live view" style="display:block;flex:0 0 220px;">{}</a>',
+            build_web_url(f"/{obj.archive_path}", request=request, config=config),
+            self._render_preview(obj, width="220px", height="138px"),
+        )
 
         return format_html(
             '<div style="display:flex; gap:16px; align-items:flex-start;">'

archivebox/core/asgi.py

@@ -7,13 +7,26 @@ For more information on this file, see
 https://docs.djangoproject.com/en/stable/howto/deployment/asgi/
 """
 
-from django.core.asgi import get_asgi_application
+from django.core.handlers.asgi import ASGIHandler
 
 from archivebox.config.django import setup_django
 
 setup_django(check_db=True)
 
-django_application = get_asgi_application()
+
+class ArchiveBoxASGIHandler(ASGIHandler):
+    def create_request(self, scope, body_file):
+        # ASGI already spools the complete body. Django's multipart parser
+        # otherwise treats chunked uploads without Content-Length as empty.
+        if not any(name == b"content-length" for name, _value in scope.get("headers", [])):
+            body_file.seek(0, 2)
+            length = body_file.tell()
+            body_file.seek(0)
+            scope = {**scope, "headers": [*scope.get("headers", []), (b"content-length", str(length).encode("ascii"))]}
+        return super().create_request(scope, body_file)
+
+
+django_application = ArchiveBoxASGIHandler()
 
 
 async def application(scope, receive, send):

archivebox/core/middleware.py

@@ -13,6 +13,7 @@ from django.http import HttpResponseForbidden, HttpResponseNotModified
 from django.shortcuts import redirect
 from django.utils import timezone
 from django.utils.http import http_date
+from django.utils.cache import patch_cache_control, patch_vary_headers
 
 from archivebox.config import VERSION
 from archivebox.config.common import get_config, get_request_config
@@ -123,13 +124,12 @@ def AdminCookieIsolationMiddleware(get_response):
 
 
 def CacheControlMiddleware(get_response):
-    snapshot_path_re = re.compile(r"^/[^/]+/\\d{8}/[^/]+/[0-9a-fA-F-]{8,36}/")
     static_cache_key = (get_COMMIT_HASH() or VERSION or "dev").strip()
 
     def middleware(request):
         response = get_response(request)
 
-        if request.path.startswith("/static/"):
+        if request.path.startswith("/static/") and response.status_code in (200, 304):
             rel_path = request.path[len("/static/") :]
             static_path = finders.find(rel_path)
             if static_path:
@@ -154,15 +154,17 @@ def CacheControlMiddleware(get_response):
                     response.headers["Last-Modified"] = http_date(mtime)
                 return response
 
-        if ("/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path)) and not response.get(
-            "Cache-Control",
-        ):
-            config = request.__dict__.get("archivebox_config")
-            if config is None:
-                config = get_config(resolve_plugins=False)
-                request.archivebox_config = config
-            policy = "private" if config.PERMISSIONS == "private" else "public"
-            response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
+        # Keep artifact freshness/validators, but never let a shared cache store
+        # private or unlisted snapshots. Auth redirects and errors stay unstored.
+        snapshot_policy = getattr(request, "archivebox_cache_policy", None)
+        if snapshot_policy and response.status_code in (200, 206, 304):
+            if not response.get("Cache-Control"):
+                response["Cache-Control"] = f"{snapshot_policy}, max-age=60, stale-while-revalidate=300"
+            if snapshot_policy == "private":
+                patch_cache_control(response, private=True)
+        elif snapshot_policy or not response.get("Cache-Control"):
+            response["Cache-Control"] = "private, no-store"
+        patch_vary_headers(response, ("Cookie", "Authorization"))
         return response
 
     return middleware

archivebox/core/models.py

@@ -473,6 +473,8 @@ class SnapshotQuerySet(models.QuerySet):
         snapshot_list = list(self.iterator(chunk_size=500))
         manifest_records = []
         for snapshot in snapshot_list:
+            # Unarchived snapshots still need an export destination before discovery.
+            snapshot.output_dir.mkdir(parents=True, exist_ok=True)
             outputs = snapshot.discover_outputs(include_filesystem_fallback=True)
             output_paths = [str(output.get("path") or "") for output in outputs]
             snapshot._public_preview_paths = [
@@ -1394,7 +1396,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         Uses full hostname with sanitized special chars.
 
         Examples:
-            https://example.com:8080 → example.com_8080
+            https://example.com:5797 → example.com_5797
             https://sub.example.com → sub.example.com
             file:///path → localhost
             data:text/html → data
@@ -1546,7 +1548,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             return cls.objects.select_related("crawl__created_by").filter(url=url, timestamp=timestamp).first()
 
     @classmethod
-    def create_from_directory(cls, snapshot_dir: Path) -> Optional["Snapshot"]:
+    def create_from_directory(cls, snapshot_dir: Path, *, crawl=None, preserve_identity: bool = False) -> Optional["Snapshot"]:
         """
         Create new Snapshot from orphaned directory.
 
@@ -1618,26 +1620,25 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             return None
 
         # Ensure uniqueness (reuses existing logic from create_or_update_from_dict)
-        timestamp = cls._ensure_unique_timestamp(url, timestamp)
+        if not preserve_identity:
+            timestamp = cls._ensure_unique_timestamp(url, timestamp)
 
         # Detect version
         fs_version = cls._detect_fs_version_from_index(data)
 
-        system_user_id = get_or_create_system_user_pk()
-        catchall_crawl, _ = Crawl.objects.get_or_create(
-            label="[migration] orphaned snapshots",
-            defaults={
-                "urls": f"# Orphaned snapshot: {url}",
-                "max_depth": 0,
-                "created_by_id": system_user_id,
-            },
-        )
-        if cls.objects.filter(crawl=catchall_crawl, url=url).exists():
+        catchall_crawl = crawl
+        if catchall_crawl is None:
+            system_user_id = get_or_create_system_user_pk()
+            catchall_crawl, _ = Crawl.objects.get_or_create(
+                label="[migration] orphaned snapshots",
+                defaults={"urls": f"# Orphaned snapshot: {url}", "max_depth": 0, "created_by_id": system_user_id},
+            )
+        if not preserve_identity and cls.objects.filter(crawl=catchall_crawl, url=url).exists():
             catchall_crawl = Crawl.objects.create(
                 label=f"[migration] orphaned snapshot {timestamp}",
                 urls=url,
                 max_depth=0,
-                created_by_id=system_user_id,
+                created_by_id=catchall_crawl.created_by_id,
             )
 
         snapshot_kwargs = {
@@ -1647,6 +1648,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             "fs_version": fs_version,
             "crawl": catchall_crawl,
         }
+        if preserve_identity:
+            snapshot_kwargs.update({key: data[key] for key in ("id", "depth", "config", "notes") if key in data})
         try:
             bookmarked_at = parse_date(data.get("bookmarked_at") or timestamp)
         except (TypeError, ValueError, OSError):
@@ -1847,7 +1850,13 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                         result_data["plugin"] = result_data.get("plugin") or result_data.get("extractor") or plugin
                         self._create_archive_result_if_missing(result_data, existing, update_existing=update_existing)
 
-    def _create_archive_result_if_missing(self, result_data: dict, existing: dict, update_existing: bool = True):
+    def _create_archive_result_if_missing(
+        self,
+        result_data: dict,
+        existing: dict,
+        update_existing: bool = True,
+        preserve_identity: bool = False,
+    ):
         """Create ArchiveResult if not already in DB."""
         from dateutil import parser
         from django.db import transaction
@@ -1925,7 +1934,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         # Machine.current() can probe the host and sanitize config. Do that before
         # atomic() so the transaction below only covers the two related row writes.
-        machine = Machine.current() if cmd or pwd else None
+        machine = Machine.current() if (cmd or pwd) and not preserve_identity else None
         with transaction.atomic():
             if machine is not None:
                 process = Process.objects.create(
@@ -1941,6 +1950,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                 )
 
             archiveresult = ArchiveResult.objects.create(
+                **({"id": result_data["id"]} if preserve_identity else {}),
                 snapshot=self,
                 plugin=plugin,
                 hook_name=hook_name,
@@ -1954,6 +1964,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                 end_ts=end_ts,
                 process=process,
             )
+            if preserve_identity:
+                dates = {key: parse_date(result_data[key]) for key in ("created_at", "modified_at") if result_data.get(key)}
+                if dates:
+                    ArchiveResult.objects.filter(pk=archiveresult.pk).update(**dates)
         existing[(plugin, hook_name)] = archiveresult
 
     def write_index_json(self):
@@ -2010,7 +2024,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         # without a corresponding DB row can be retained byte-for-byte.
         binaries_seen = set()
         processes_seen = set()
-        records = [self.to_json()]
+        records = [self.to_json(), self.crawl.to_json()]
         for ar in archive_results:
             process = ar.process_record
             if process and process.binary and process.binary_id not in binaries_seen:
@@ -2837,6 +2851,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         Includes all fields needed to fully reconstruct/identify this snapshot.
         """
         from archivebox.config import VERSION
+        from archivebox.config.common import redact_sensitive_config
 
         archive_size = self.archive_size
 
@@ -2850,6 +2865,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             "tags": self.tags_str(),
             "bookmarked_at": self.bookmarked_at.isoformat() if self.bookmarked_at else None,
             "created_at": self.created_at.isoformat() if self.created_at else None,
+            "modified_at": self.modified_at.isoformat() if self.modified_at else None,
+            "config": redact_sensitive_config(self.config),
+            "notes": self.notes,
             "timestamp": self.timestamp,
             "depth": self.depth,
             "status": self.status,
@@ -3262,12 +3280,23 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         self,
         include_filesystem_fallback: bool = True,
         archive_results: list["ArchiveResult"] | None = None,
+        filesystem_index: dict | None = None,
     ) -> list[dict]:
-        """Discover output files from ArchiveResults and filesystem."""
+        """Discover output paths; file availability is not result success.
+
+        Live cards must take succeeded/failed state from ArchiveResult.status.
+        A failed hook can leave useful partial files, and hooks/retries share
+        plugin directories, so an embeddable path or nonzero size is not proof
+        of success. Filesystem discovery also supports portable static exports;
+        it must not overwrite or replace the live database's lifecycle facts.
+
+        This includes partial files for browsing. get_html_details_context
+        separately admits successful DB results to the output cards.
+        """
         from archivebox.misc.util import ts_to_date_str
 
         ArchiveResult = self.archiveresult_set.model
-        snap_dir = Path(self.output_dir)
+        snap_dir = Path(self.output_dir) if include_filesystem_fallback else None
         outputs: list[dict] = []
         seen: set[str] = set()
 
@@ -3281,13 +3310,23 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             lower = (path or "").lower()
             return lower.endswith(text_exts)
 
-        hashes_index = self.hashes_index if include_filesystem_fallback else {}
+        if filesystem_index is not None:
+            filesystem_index.update(ArchiveResult._scan_output_file_map(snap_dir))
+        hashes_index = filesystem_index if filesystem_index is not None else (self.hashes_index if include_filesystem_fallback else {})
+        if include_filesystem_fallback and not hashes_index and filesystem_index is None:
+            hashes_index = ArchiveResult._scan_output_file_map(snap_dir)
         results = archive_results if archive_results is not None else self.archiveresult_set.all().order_by("start_ts")
         for result in results:
             output_file_map = result.output_file_map()
             embed_path = result.embed_path_db(output_file_map=output_file_map)
+            if include_filesystem_fallback and embed_path not in hashes_index:
+                raw_output = str(result.output_str or "")
+                embed_path = raw_output if raw_output in hashes_index else None
             if not embed_path and include_filesystem_fallback:
-                embed_path = result.embed_path()
+                prefix = f"{result.plugin}/"
+                plugin_files = {path.removeprefix(prefix): meta for path, meta in hashes_index.items() if path.startswith(prefix)}
+                fallback = ArchiveResult._fallback_output_file_path(list(plugin_files), result.plugin, plugin_files)
+                embed_path = f"{prefix}{fallback}" if fallback else None
             if not embed_path or embed_path.strip() in (".", "/", "./"):
                 continue
             size = (
@@ -3296,24 +3335,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                 or hashes_index.get(embed_path, {}).get("size")
                 or 0
             )
-            if not size and include_filesystem_fallback and not hashes_index:
-                abs_path = snap_dir / embed_path
-                if not abs_path.exists():
-                    continue
-                if abs_path.is_dir():
-                    if not any(p.is_file() for p in abs_path.rglob("*")):
-                        continue
-                    size = sum(p.stat().st_size for p in abs_path.rglob("*") if p.is_file())
-                else:
-                    size = abs_path.stat().st_size
-                    plugin_lower = (result.plugin or "").lower()
-                    if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl"):
-                        plugin_dir = snap_dir / result.plugin
-                        if plugin_dir.exists():
-                            try:
-                                size = sum(p.stat().st_size for p in plugin_dir.rglob("*") if p.is_file())
-                            except OSError:
-                                pass
             outputs.append(
                 {
                     "name": result.plugin,
@@ -3345,13 +3366,12 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                 fallback_path = ArchiveResult._fallback_output_file_path(list(root_entries.keys()), root, root_entries)
                 if not fallback_path or not (snap_dir / root / fallback_path).exists():
                     continue
-                fallback_meta = root_entries.get(fallback_path, {})
                 outputs.append(
                     {
                         "name": root,
                         "path": f"{root}/{fallback_path}",
                         "ts": fallback_ts,
-                        "size": int(fallback_meta.get("size") or 0),
+                        "size": sum(int(meta.get("size") or 0) for meta in root_entries.values() if not meta.get("is_dir")),
                         "is_metadata": is_metadata_path(fallback_path),
                         "is_compact": is_compact_path(fallback_path),
                         "result": None,
@@ -3359,9 +3379,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                 )
                 seen.add(root)
 
-        if not include_filesystem_fallback or hashes_index:
-            return outputs
-        if not snap_dir.is_dir():
+        if not include_filesystem_fallback:
             return outputs
 
         embeddable_exts = {
@@ -3391,50 +3409,26 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             "wav",
         }
 
-        for entry in snap_dir.iterdir():
-            if entry.name in ("index.html", "index.json", "favicon.ico", "warc"):
+        for path, metadata in hashes_index.items():
+            entry = Path(path)
+            if len(entry.parts) != 1 or entry.name in ("index.html", "index.json", "favicon.ico"):
                 continue
-            if entry.is_dir():
-                plugin = entry.name
-                if plugin in seen:
-                    continue
-                best_file = ArchiveResult._find_best_output_file(entry, plugin)
-                if not best_file:
-                    continue
-                best_file_stat = best_file.stat()
-                rel_path = str(best_file.relative_to(snap_dir))
-                outputs.append(
-                    {
-                        "name": plugin,
-                        "path": rel_path,
-                        "ts": ts_to_date_str(best_file_stat.st_mtime or 0),
-                        "size": best_file_stat.st_size or 0,
-                        "is_metadata": is_metadata_path(rel_path),
-                        "is_compact": is_compact_path(rel_path),
-                        "result": None,
-                    },
-                )
-                seen.add(plugin)
-            elif entry.is_file():
-                ext = entry.suffix.lstrip(".").lower()
-                if ext not in embeddable_exts:
-                    continue
-                plugin = entry.stem
-                if plugin in seen:
-                    continue
-                entry_stat = entry.stat()
-                outputs.append(
-                    {
-                        "name": plugin,
-                        "path": entry.name,
-                        "ts": ts_to_date_str(entry_stat.st_mtime or 0),
-                        "size": entry_stat.st_size or 0,
-                        "is_metadata": is_metadata_path(entry.name),
-                        "is_compact": is_compact_path(entry.name),
-                        "result": None,
-                    },
-                )
-                seen.add(plugin)
+            if filesystem_index is not None and entry.name == "index.jsonl":
+                continue
+            if entry.suffix.lstrip(".").lower() not in embeddable_exts or entry.stem in seen:
+                continue
+            outputs.append(
+                {
+                    "name": entry.stem,
+                    "path": path,
+                    "ts": ts_to_date_str(self.downloaded_at or self.created_at),
+                    "size": metadata.get("size", 0),
+                    "is_metadata": is_metadata_path(path),
+                    "is_compact": is_compact_path(path),
+                    "result": None,
+                },
+            )
+            seen.add(entry.stem)
 
         return outputs
 
@@ -3514,13 +3508,20 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         path = output_dir / CONSTANTS.JSON_INDEX_FILENAME
         atomic_write(str(path), self.to_dict(extended=True, static_export=True))
 
-    def get_html_details_context(self, request=None, *, static_export_dir: Path | None = None) -> dict[str, Any]:
+    def get_html_details_context(
+        self,
+        request=None,
+        *,
+        static_export_dir: Path | None = None,
+        discover_files: bool = False,
+    ) -> dict[str, Any]:
         """Build the one context used by both served and on-disk snapshot pages."""
         from archivebox.config.common import get_request_config
         from archivebox.core.permissions import get_snapshot_permissions
         from archivebox.core.widgets import TagEditorWidget
         from archivebox.misc.logging_util import printable_filesize
         from archivebox.progressmonitor.views import progress_endpoint
+        from archivebox.plugins.output_groups import OUTPUT_GROUPS, order_snapshot_outputs
 
         runtime_config = get_request_config(request) if request is not None else get_config()
         self._runtime_config = runtime_config
@@ -3532,11 +3533,24 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         self.__dict__["num_outputs_cached"] = sum(result.status == ArchiveResult.StatusChoices.SUCCEEDED for result in archive_results)
         self.__dict__["num_failures_cached"] = sum(result.status == ArchiveResult.StatusChoices.FAILED for result in archive_results)
 
-        hidden_card_plugins = {"archivedotorg", "favicon", "title"}
+        filesystem_index = {} if discover_files else None
+        hidden_card_plugins = {"archivedotorg", "favicon"}
         outputs = [
             output
-            for output in self.discover_outputs(include_filesystem_fallback=True, archive_results=archive_results)
-            if (output.get("size") or 0) > 0 and output.get("name") not in hidden_card_plugins
+            for output in self.discover_outputs(
+                include_filesystem_fallback=static_export_dir is not None or filesystem_index is not None,
+                archive_results=archive_results,
+                filesystem_index=filesystem_index,
+            )
+            if (output.get("size") or 0) > 0
+            and output.get("name") not in hidden_card_plugins
+            # Files left by failed/started/noresults hooks remain browsable,
+            # but only the DB can authorize a successful live output card.
+            and (
+                output["result"].status == ArchiveResult.StatusChoices.SUCCEEDED
+                if output.get("result") is not None
+                else static_export_dir is not None
+            )
         ]
         if static_export_dir is not None:
 
@@ -3556,34 +3570,30 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             current = outputs_by_name.get(output["name"])
             if current is None or (output.get("size") or 0) > (current.get("size") or 0):
                 outputs_by_name[output["name"]] = output
+        # Several hooks can report the same plugin directory. Count overlapping
+        # files once, while retaining every result ID for the existing delete action.
+        files_by_name: dict[str, dict[str, int]] = {}
+        for result in archive_results:
+            if result.plugin not in outputs_by_name:
+                continue
+            files = files_by_name.setdefault(result.plugin, {})
+            for path, metadata in result.output_file_map().items():
+                files[path] = result._coerce_output_file_size(metadata.get("size"))
         for name, output in outputs_by_name.items():
             output["result_ids"] = ",".join(result_ids_by_name.get(name, ()))
+            if files_by_name.get(name):
+                output["size"] = sum(files_by_name[name].values())
 
-        hash_index = self.hashes_index
+        hash_index = filesystem_index if filesystem_index is not None else (self.hashes_index if static_export_dir is not None else {})
         loose_items, failed_items = self.get_detail_page_auxiliary_items(
             outputs,
             hidden_card_plugins=hidden_card_plugins,
             archive_results=archive_results,
+            hashes_index=hash_index,
         )
-        preview_priority = ("singlefile", "screenshot", "wget", "dom", "pdf", "readability")
-        output_order = {result_type: index for index, result_type in enumerate(outputs_by_name)}
-        ordered_outputs = sorted(
-            outputs_by_name.values(),
-            key=lambda output: (
-                preview_priority.index(output["name"]) if output["name"] in preview_priority else len(preview_priority),
-                output_order.get(output["name"], len(output_order)),
-            ),
-        )
-        best_result = {"path": "about:blank", "result": None}
-        for result_type in preview_priority:
-            if result_type in outputs_by_name:
-                best_result = outputs_by_name[result_type]
-                break
-        if best_result["path"] == "about:blank" and ordered_outputs:
-            best_result = ordered_outputs[0]
+        ordered_outputs = order_snapshot_outputs(list(outputs_by_name.values()))
+        best_result = ordered_outputs[0] if ordered_outputs else {"path": "about:blank", "result": None}
 
-        non_compact_outputs = [output for output in ordered_outputs if not output.get("is_compact") and not output.get("is_metadata")]
-        compact_outputs = [output for output in ordered_outputs if output.get("is_compact") or output.get("is_metadata")]
         archive_dates = [result.start_ts for result in archive_results if result.start_ts]
         output_size = sum(int(output.get("size") or 0) for output in ordered_outputs)
         has_outputs = bool(ordered_outputs)
@@ -3658,7 +3668,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             "num_failures": self.num_failures,
             "oldest_archive_date": ts_to_date_str(min(archive_dates) if archive_dates else None),
             "warc_path": warc_path,
-            "archiveresults": [*non_compact_outputs, *compact_outputs],
+            "archiveresults": ordered_outputs,
+            "output_groups": [{"id": group_id, "label": label} for group_id, label, _ in OUTPUT_GROUPS],
             "best_result": best_result,
             "snapshot": self,
             "CONFIG": runtime_config,
@@ -3691,6 +3702,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         outputs: list[dict] | None = None,
         hidden_card_plugins: set[str] | None = None,
         archive_results: list["ArchiveResult"] | None = None,
+        hashes_index: dict[str, dict[str, Any]] | None = None,
     ) -> tuple[list[dict[str, object]], list[dict[str, object]]]:
         if outputs is None:
             outputs = self.discover_outputs(include_filesystem_fallback=True)
@@ -3709,9 +3721,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         ignore_names = {".DS_Store", "index.html", "index.json", "index.jsonl", "favicon.ico"}
         loose_items: list[dict[str, object]] = []
-        if self.hashes_index:
+        hashes_index = self.hashes_index if hashes_index is None else hashes_index
+        if hashes_index:
             grouped: dict[str, dict[str, object]] = {}
-            for rel_path, meta in self.hashes_index.items():
+            for rel_path, meta in hashes_index.items():
                 parts = Path(rel_path).parts
                 if not parts:
                     continue
@@ -4009,6 +4022,8 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
             "schema_version": VERSION,
             "id": str(self.id),
             "snapshot_id": str(self.snapshot_id),
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+            "modified_at": self.modified_at.isoformat() if self.modified_at else None,
             "plugin": self.plugin,
             "hook_name": self.hook_name,
             "status": self.status,
@@ -4222,7 +4237,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
     def output_file_paths(self) -> list[str]:
         return list(self.output_file_map().keys())
 
-    def update_output_metadata_from_filesystem(self, snapshot_dir: Path | None = None, save: bool = True) -> bool:
+    def update_output_metadata_from_filesystem(self, snapshot_dir: Path | None = None, save: bool = True, full_scan: bool = False) -> bool:
         from abx_dl.output_files import OutputManifest, output_file_from_path
 
         if self.plugin == "title":
@@ -4262,7 +4277,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
                 add_file(snapshot_dir / raw_output, raw_output, root_relative=True)
 
         plugin_dir = snapshot_dir / self.plugin
-        if not output_files and plugin_dir.is_dir():
+        if (full_scan or not output_files) and plugin_dir.is_dir():
             output_files = OutputManifest.scan(plugin_dir, containment_root=snapshot_dir).as_mapping()
 
         if not output_files:
@@ -4410,26 +4425,32 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
 
         return None
 
+    @staticmethod
+    def _scan_output_file_map(dir_path: Path, *, max_scan: int | None = None) -> dict[str, dict[str, Any]]:
+        """Collect read-only metadata for the existing output preview selector."""
+        file_map: dict[str, dict[str, Any]] = {}
+
+        def scan_error(error):
+            raise error
+
+        scanned = 0
+        for parent, directories, filenames in os.walk(dir_path, onerror=scan_error, followlinks=False):
+            directories[:] = [name for name in directories if not name.startswith(".")]
+            for name in filenames:
+                scanned += 1
+                if max_scan is not None and scanned > max_scan:
+                    return file_map
+                file_path = Path(parent) / name
+                if file_path.is_symlink() or name.startswith("."):
+                    continue
+                file_map[str(file_path.relative_to(dir_path))] = {"size": file_path.stat().st_size}
+        return file_map
+
     @staticmethod
     def _find_best_output_file(dir_path: Path, plugin_name: str | None = None) -> Path | None:
         if not dir_path.exists() or not dir_path.is_dir():
             return None
-        file_map: dict[str, dict[str, Any]] = {}
-        file_count = 0
-        max_scan = 500
-        for file_path in dir_path.rglob("*"):
-            file_count += 1
-            if file_count > max_scan:
-                break
-            if file_path.is_dir() or file_path.name.startswith("."):
-                continue
-            rel_path = str(file_path.relative_to(dir_path))
-            try:
-                size = file_path.stat().st_size
-            except OSError:
-                size = 0
-            file_map[rel_path] = {"size": size}
-
+        file_map = ArchiveResult._scan_output_file_map(dir_path, max_scan=500)
         fallback_path = ArchiveResult._fallback_output_file_path(list(file_map.keys()), plugin_name, file_map)
         if not fallback_path:
             return None
@@ -4459,7 +4480,17 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
                     candidates.append(raw_output)
 
                 if not output_file_map:
-                    return self._existing_output_path(raw_output)
+                    from ipaddress import ip_address
+
+                    try:
+                        ip_address(raw_output)
+                    except ValueError:
+                        pass
+                    else:
+                        return None
+                    # Legacy rows still have a stored display path. Do not probe
+                    # payload storage while rendering a live snapshot page.
+                    return candidates[0] if ".." not in output_path.parts else None
 
                 if raw_output in output_file_map and is_root_relative(raw_output):
                     return raw_output

archivebox/core/preview_util.py

@@ -1,4 +1,114 @@
 __package__ = "archivebox.core"
 
+import hashlib
+import json
+from pathlib import PurePosixPath
+from urllib.parse import urlsplit
+
+from django.template.loader import render_to_string
+from django.utils.safestring import mark_safe
 
 EXTENSION_SCREENSHOT_PLUGIN = "chrome_extension_screenshot"
+PREVIEW_PLUGINS = ("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "seo", "responses", "favicon")
+IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".gif", ".avif", ".svg", ".ico"}
+
+
+def positive_int(value):
+    try:
+        return max(0, int(value))
+    except (ValueError, TypeError, OverflowError):
+        return 0
+
+
+def preview_candidates(results):
+    """Select only saved outputs. Never read files or fetch remote metadata."""
+    candidates = []
+    for result in results:
+        files = result.output_files
+        if not isinstance(files, dict):
+            continue
+        for path, info in files.items():
+            if not isinstance(path, str) or not isinstance(info, dict) or not positive_int(info.get("size")):
+                continue
+            if path.startswith("/") or ".." in PurePosixPath(path).parts:
+                continue
+            plugin = result.plugin
+            relative = path.removeprefix(f"{plugin}/")
+            output_path = path if info.get("root_relative") else f"{plugin}/{path}"
+            kind = "image"
+            rank = None
+            if plugin == "screenshot" and relative == "screenshot.png":
+                rank = 0
+            elif plugin == EXTENSION_SCREENSHOT_PLUGIN and relative in ("screenshot-1.png", "screenshot.png"):
+                rank = 1 if relative == "screenshot-1.png" else 2
+            elif (
+                plugin == "seo"
+                and PurePosixPath(relative).stem == "featured-image"
+                and PurePosixPath(relative).suffix.lower() in IMAGE_EXTENSIONS
+            ):
+                rank = 3
+            elif plugin == "responses":
+                mime = str(info.get("mimetype") or info.get("mime_type") or "")
+                if not (relative.startswith("image/") or (relative.startswith("all/") and mime.startswith("image/"))):
+                    continue
+                width, height = positive_int(info.get("width")), positive_int(info.get("height"))
+                if width and height and (min(width, height) <= 2 or max(width, height) <= 64):
+                    continue
+                if PurePosixPath(relative).suffix.lower() == ".ico":
+                    continue
+                rank = 4
+                kind = "response"
+            elif plugin == "favicon" and relative == "favicon.ico":
+                rank = 5
+                kind = "favicon"
+            if rank is None:
+                continue
+            width, height = positive_int(info.get("width")), positive_int(info.get("height"))
+            # Prefer usable known dimensions; byte size remains the legacy fallback.
+            usable = bool(width and height and 0.25 <= width / height <= 4)
+            score = (rank, -int(usable), -(width * height if usable else 0), -positive_int(info.get("size")), output_path)
+            candidates.append((score, {"path": output_path, "kind": kind, "plugin": plugin, "filename": path}))
+    seen = set()
+    selected = []
+    for _score, candidate in sorted(candidates, key=lambda item: item[0]):
+        if candidate["path"] not in seen:
+            selected.append(candidate)
+            seen.add(candidate["path"])
+    return selected
+
+
+def snapshot_preview_candidates(snapshot):
+    if "_preview_candidates" not in snapshot.__dict__:
+        results = snapshot.__dict__.get("_preview_results")
+        if results is None:
+            results = snapshot.__dict__.get("_prefetched_objects_cache", {}).get("archiveresult_set")
+        if results is None:
+            results = snapshot.archiveresult_set.filter(plugin__in=PREVIEW_PLUGINS).only("plugin", "output_files")
+        snapshot._preview_candidates = preview_candidates(results)
+    return snapshot._preview_candidates
+
+
+def render_snapshot_preview(snapshot, url_for_candidate, width="100px", height="100px"):
+    candidates = []
+    for candidate in snapshot_preview_candidates(snapshot):
+        candidates.append({"url": url_for_candidate(candidate), "kind": candidate["kind"]})
+    try:
+        host = urlsplit(snapshot.url).hostname or "Archive"
+    except ValueError:
+        host = "Archive"
+    host = host.removeprefix("www.")
+    hue = int(hashlib.sha256(host.encode()).hexdigest()[:6], 16) % 360
+    return mark_safe(
+        render_to_string(
+            "core/snapshot_thumbnail.html",
+            {
+                "candidates": json.dumps(candidates),
+                "first": candidates[0] if candidates else None,
+                "host": host,
+                "initial": host[:1].upper(),
+                "hue": hue,
+                "width": width,
+                "height": height,
+            },
+        ),
+    )

archivebox/core/recovery_util.py

@@ -140,6 +140,13 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
             orphaned_hook_processes = orphaned_hook_processes.filter(snapshot_pwd_filter)
         else:
             orphaned_hook_processes = orphaned_hook_processes.none()
+    else:
+        # Global startup recovery must stay database-only. Process.pwd points
+        # into archive payload storage, which may be a cold FUSE mount; scanning
+        # every historical unlinked hook here blocks the sole scheduler before
+        # it can claim newly queued work. Explicit crawl recovery below remains
+        # available for the work the caller asked to resume.
+        orphaned_hook_processes = orphaned_hook_processes.none()
     for process in orphaned_hook_processes.only(
         "id",
         "pwd",

archivebox/core/routes_util.py

@@ -74,7 +74,7 @@ def derive_base_url_from_csrf(config: dict[str, Any] | None = None, **config_kwa
     had ``CSRF_TRUSTED_ORIGINS=https://archive.example.com`` set (required for
     admin login to work). On upgrade, ``BASE_URL`` is the new knob — but it
     defaults to empty, and falling through to ``BIND_ADDR`` produces an
-    unreachable URL like ``http://0.0.0.0:8000``. If the user has exactly one
+    unreachable URL like ``http://0.0.0.0:5797``. If the user has exactly one
     CSRF origin we treat it as the implicit BASE_URL so links/redirects keep
     pointing at the public hostname they already configured.
 
@@ -133,7 +133,7 @@ def canonical_base_host_for_request(request_host: str) -> str:
     """Strip role subdomains and remap loopback hostnames to ``archivebox.localhost``.
 
     Used by the banner suggestion and the in-browser pin endpoint: when the
-    user is hitting the server on raw ``localhost:9292`` or ``127.0.0.1:9292``
+    user is hitting the server on raw ``localhost:5797`` or ``127.0.0.1:5797``
     we want to suggest the wildcard-friendly ``archivebox.localhost`` family
     instead, so the eventual pinned ``BASE_URL`` plays nicely with subdomain
     routing without forcing the user to add a /etc/hosts entry.

archivebox/core/settings.py

@@ -87,6 +87,7 @@ DJANGO_OBJECT_ACTIONS_DEFAULT_HTTP_METHOD = "POST"
 MIDDLEWARE = [
     "archivebox.core.middleware.TimezoneMiddleware",
     "django.middleware.security.SecurityMiddleware",
+    "archivebox.core.middleware.CacheControlMiddleware",
     "archivebox.core.middleware.AdminCookieIsolationMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.common.CommonMiddleware",
@@ -97,7 +98,6 @@ MIDDLEWARE = [
     "archivebox.core.middleware.ServerSecurityModeMiddleware",
     "archivebox.core.middleware.HostRoutingMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
-    "archivebox.core.middleware.CacheControlMiddleware",
     # Additional middlewares from plugins (if any)
 ]
 
@@ -390,7 +390,7 @@ SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
 # get_base_url() also covers deployments that only set CSRF_TRUSTED_ORIGINS (the
 # implicit-BASE_URL fallback used on 0.7.x->0.9.x upgrades), so HTTPS hardening
 # isn't lost until BASE_URL is migrated. An explicit plain-http base (e.g. local
-# http://archivebox.localhost:8000) disables proxy HTTPS handling.
+# http://archivebox.localhost:5797) disables proxy HTTPS handling.
 BASE_URL_IS_HTTPS = get_base_url(config=CONFIG).strip().lower().startswith("https://")
 if BASE_URL_IS_HTTPS or not CONFIG.BASE_URL:
     SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")

archivebox/core/shutdown_util.py

@@ -101,6 +101,7 @@ def foreground_shutdown_signals(
     *,
     first_signal_message: str | None = "\n[🛑] Got {signal_name}, stopping gracefully...\n",
     on_signal: Callable[[signal.Signals], None] | None = None,
+    interrupt_handlers: dict[signal.Signals, Callable[[], None]] | None = None,
     raise_on_first_signal: bool = True,
 ) -> Iterator[ShutdownSignalState]:
     """Install foreground signal handlers that print an immediate exit notice.
@@ -118,10 +119,20 @@ def foreground_shutdown_signals(
 
     state = ShutdownSignalState()
     previous_active_state = _active_shutdown_state
-    previous_handlers = {sig: signal.getsignal(sig) for sig in handled_signals}
+    previous_handlers = {sig: signal.getsignal(sig) for sig in (*handled_signals, *(interrupt_handlers or {}))}
 
     def raise_keyboard_interrupt(signum, _frame):
         sig = signal.Signals(signum)
+        if interrupt_handlers and sig in interrupt_handlers:
+            # Shield resumable terminal intent from sticky shutdown state.
+            # Ctrl+C is delegated to abx-dl's pause/abort controller; SIGUSR1/2
+            # carry prompt answers. Recording any of these as shutdown here
+            # makes a later skip/retry fail raise_if_shutdown_requested(), or
+            # makes a second Ctrl+C hard-exit before hooks can flush output.
+            # SIGTERM/SIGHUP deliberately bypass this shield: ownership takeover
+            # and parent loss must cancel the old worker without asking for input.
+            interrupt_handlers[sig]()
+            return
         already_requested = state.signal_name is not None
         if not already_requested:
             state.signal_name = sig.name
@@ -129,11 +140,9 @@ def foreground_shutdown_signals(
                 os.write(sys.stdout.fileno(), first_signal_message.format(signal_name=state.signal_name).encode())
         if on_signal is not None:
             on_signal(sig)
-        # Foreground `archivebox add` uses the first signal to abort the active
-        # hook through the bus-facing runner code, then reserves the second
-        # signal for hard foreground-command shutdown. Server/update/run and
-        # other non-interactive commands raise immediately so their finally
-        # blocks can stop owned children without prompting.
+        # Actual shutdown requests remain sticky and a second signal forces
+        # exit. Resumable interactive actions were handled above without
+        # marking this context as shutting down.
         if already_requested:
             os.write(sys.stdout.fileno(), f"\n[🛑] Got {sig.name} again, exiting immediately.\n".encode())
             os._exit(130)
@@ -142,7 +151,7 @@ def foreground_shutdown_signals(
 
     try:
         _active_shutdown_state = state
-        for sig in handled_signals:
+        for sig in previous_handlers:
             signal.signal(sig, raise_keyboard_interrupt)
         yield state
     finally:

archivebox/core/templatetags/core_tags.py

@@ -7,7 +7,7 @@ from urllib.parse import quote, urlparse
 
 from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target
 from django import template
-from django.templatetags.static import static
+from django.urls import reverse
 from django.utils import timezone
 from django.utils.html import escape
 from django.utils.safestring import mark_safe
@@ -22,6 +22,7 @@ from archivebox.core.routes_util import (
 )
 from archivebox.core.setup_wizard import get_base_url_mismatch_context, get_setup_wizard_context
 from archivebox.plugins.discovery import (
+    get_plugin_catalog,
     get_plugin_icon,
     get_plugin_name,
     get_plugin_template,
@@ -154,10 +155,10 @@ def _count_media_files(result) -> int:
     return count
 
 
-def _list_media_files(result) -> list[dict]:
+def _list_media_files(result, *, include_filesystem_fallback=True) -> list[dict]:
     media_files: list[dict] = []
     try:
-        plugin_dir = Path(result.snapshot_dir) / result.plugin
+        plugin_dir = Path(result.snapshot_dir) / result.plugin if include_filesystem_fallback else None
     except (AttributeError, TypeError, ValueError):
         return media_files
 
@@ -169,7 +170,7 @@ def _list_media_files(result) -> list[dict]:
             if rel_path.suffix.lower() in _MEDIA_FILE_EXTS:
                 candidates.append((rel_path, _coerce_output_file_size(metadata.get("size"))))
 
-    if not candidates and plugin_dir.exists():
+    if not candidates and plugin_dir is not None and plugin_dir.exists():
         scanned = 0
         max_scan = 2000
         for file_path in plugin_dir.rglob("*"):
@@ -329,7 +330,7 @@ def _render_text_preview(plugin: str, icon_html: str, snippet: str) -> str:
     plugin_label = escape(plugin or "")
     escaped = escape(snippet)
     return (
-        f'<div class="thumbnail-text" data-plugin="{plugin_attr}" data-compact="1">'
+        f'<div class="thumbnail-text" data-plugin="{plugin_attr}">'
         f'<div class="thumbnail-text-header">'
         f'<span class="thumbnail-compact-icon">{icon_html}</span>'
         f'<span class="thumbnail-text-title">{plugin_label}</span>'
@@ -344,7 +345,7 @@ def _render_fallback_card(plugin: str, icon_html: str, fallback_label: str) -> s
     plugin_label = escape(plugin or "")
     fallback_attr = escape(fallback_label)
     return (
-        f'<div class="thumbnail-compact" data-plugin="{plugin_attr}" data-compact="1">'
+        f'<div class="thumbnail-compact" data-plugin="{plugin_attr}">'
         f'<span class="thumbnail-compact-icon">{icon_html}</span>'
         f'<span class="thumbnail-compact-label">{plugin_label}</span>'
         f'<span class="thumbnail-compact-meta">{fallback_attr}</span>'
@@ -384,12 +385,12 @@ def index(value, position):
 
 
 @register.filter
-def file_size(num_bytes: float) -> str:
+def file_size(num_bytes: float, decimal_places: int = 1) -> str:
     for count in ["Bytes", "KB", "MB", "GB"]:
         if num_bytes > -1024.0 and num_bytes < 1024.0:
-            return f"{num_bytes:3.1f} {count}"
+            return f"{num_bytes:.{decimal_places}f} {count}"
         num_bytes /= 1024.0
-    return "{:3.1f} {}".format(num_bytes, "TB")
+    return f"{num_bytes:.{decimal_places}f} TB"
 
 
 @register.filter
@@ -460,9 +461,9 @@ def system_warnings_banner(context):
     """Render the top-of-page warning banner for one of the conditions below,
     in priority order (highest first):
 
-    1. ``mode="unconfigured"``— ``BASE_URL`` is empty. Security/correctness
-       issue: until it's pinned, generated URLs can echo any Host the client
-       sends, and admin/web/api routing has no canonical anchor.
+    1. ``mode="unconfigured"``— ``BASE_URL`` is empty and the operator has
+       not selected safe single-domain routing. That mode intentionally uses
+       each device's connection address, including LAN and Tailscale IPs.
     2. ``mode="base_url_mismatch"`` — the browser reached ArchiveBox through
        an origin that differs from the configured canonical ``BASE_URL``.
     3. ``mode="unsafe"``      — ``SERVER_SECURITY_MODE`` explicitly enables
@@ -489,7 +490,7 @@ def system_warnings_banner(context):
 
         config = get_config(resolve_plugins=False)
 
-    if not config.BASE_URL:
+    if not config.BASE_URL and config.SERVER_SECURITY_MODE != "safe-onedomain-nojsreplay":
         return get_setup_wizard_context(context.get("request"), config)
     mismatch = get_base_url_mismatch_context(context.get("request"), config)
     if mismatch and not _is_web_subdomain_alias(mismatch["browser_url"], mismatch["configured_base_url"]):
@@ -613,6 +614,27 @@ def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) ->
     return ""
 
 
+@register.simple_tag(takes_context=True)
+def admin_snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> str:
+    if not snapshot_archiveresult_url(context, snapshot, plugin, filename):
+        return ""
+    return reverse("admin:core_snapshot_preview", args=(snapshot.pk, plugin, filename))
+
+
+@register.simple_tag(takes_context=True)
+def snapshot_thumbnail(context, snapshot, admin=False, width="100px", height="100px"):
+    from archivebox.core.preview_util import render_snapshot_preview
+
+    if admin:
+        url_for_candidate = lambda candidate: reverse(
+            "admin:core_snapshot_preview",
+            args=(snapshot.pk, candidate["plugin"], candidate["filename"]),
+        )
+    else:
+        url_for_candidate = lambda candidate: _snapshot_url_for_context(context, snapshot, candidate["path"])
+    return render_snapshot_preview(snapshot, url_for_candidate, width=width, height=height)
+
+
 @register.simple_tag(takes_context=True)
 def snapshot_index_row(context, link) -> str:
     snapshot_base = _snapshot_base_url_for_context(context, link)
@@ -634,7 +656,6 @@ def snapshot_index_row(context, link) -> str:
 
     url = getattr(link, "url", "") or ""
     title = unescape(getattr(link, "title", "") or "")
-    is_pending = status in {"queued", "started", "backoff"}
     title_text = title or url
     tags_str = link.tags_str() if callable(getattr(link, "tags_str", None)) else getattr(link, "tags_str", "")
     tag_html = "".join(f'<span class="snapshot-tag">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag)
@@ -654,43 +675,7 @@ def snapshot_index_row(context, link) -> str:
     output_plural = "" if num_outputs == 1 else "s"
     files_url = _snapshot_url_for_context(context, link, "index.jsonl") if context.get("STATIC_EXPORT") else f"{snapshot_base}/?files=1"
 
-    if is_pending:
-        preview_html = '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress"></span>'
-        if not context.get("STATIC_EXPORT"):
-            preview_html = (
-                '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress">'
-                f'<img src="{escape(static("spinner.gif"))}" alt="" decoding="async" loading="lazy">'
-                "</span>"
-            )
-    elif "_public_preview_paths" in link.__dict__:
-        preview_paths = list(getattr(link, "_public_preview_paths", []) or [])
-        if preview_paths:
-            preview_urls = [_snapshot_url_for_context(context, link, path) for path in preview_paths]
-            preview_html = (
-                f'<img src="{escape(preview_urls[0])}" '
-                f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
-                'onerror="nextPublicSnapshotPreview(this)" class="snapshot-preview screenshot" alt="" decoding="async" loading="lazy">'
-            )
-        else:
-            preview_html = '<span class="snapshot-preview snapshot-preview-empty" aria-label="No preview available"></span>'
-    else:
-        preview_urls = [
-            url
-            for url in (
-                snapshot_archiveresult_url(context, link, "screenshot", "screenshot.png"),
-                snapshot_archiveresult_url(context, link, "chrome_extension_screenshot", "screenshot-1.png"),
-                snapshot_archiveresult_url(context, link, "chrome_extension_screenshot", "screenshot.png"),
-            )
-            if url
-        ]
-        if preview_urls:
-            preview_html = (
-                f'<img src="{escape(preview_urls[0])}" '
-                f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
-                'onerror="nextPublicSnapshotPreview(this)" class="snapshot-preview screenshot" alt="" decoding="async" loading="lazy">'
-            )
-        else:
-            preview_html = '<span class="snapshot-preview snapshot-preview-empty" aria-label="No preview available"></span>'
+    preview_html = snapshot_thumbnail(context, link)
 
     if "_public_favicon_paths" in link.__dict__:
         favicon_paths = list(getattr(link, "_public_favicon_paths", []) or [])
@@ -812,6 +797,7 @@ def plugin_card(context, result) -> str:
         return ""
 
     plugin = get_plugin_name(result.plugin)
+    has_card_template = bool(get_plugin_template(plugin, "card", fallback=False))
     template_str = get_plugin_template(plugin, "card")
 
     # Use embed_path() for the display path
@@ -820,8 +806,12 @@ def plugin_card(context, result) -> str:
 
     icon_html = get_plugin_icon(plugin)
     plugin_lower = (plugin or "").lower()
-    media_file_count = _count_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else 0
-    media_files = _list_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
+    media_files = (
+        _list_media_files(result, include_filesystem_fallback=bool(context.get("STATIC_EXPORT")))
+        if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl")
+        else []
+    )
+    media_file_count = len(media_files)
     if context.get("STATIC_EXPORT") and media_files:
         media_files = [
             item
@@ -840,12 +830,22 @@ def plugin_card(context, result) -> str:
     output_lower = (raw_output_path or "").lower()
     force_text_preview = output_lower.endswith(_TEXT_PREVIEW_EXTS)
 
+    # Static exports cannot execute live preview routes; keep their text snippets.
+    if context.get("STATIC_EXPORT") and force_text_preview:
+        template_str = None
+
     # Create a mini template and render it with context
     try:
-        if template_str and raw_output_path and str(raw_output_path).strip() not in (".", "/", "./") and not force_text_preview:
+        if (
+            template_str
+            and raw_output_path
+            and str(raw_output_path).strip() not in (".", "/", "./")
+            and (has_card_template or not force_text_preview)
+        ):
             tpl = template.Template(template_str)
             ctx = template.Context(
                 {
+                    **context.flatten(),
                     "result": result,
                     "snapshot": result.snapshot,
                     "output_path": output_url,
@@ -863,7 +863,23 @@ def plugin_card(context, result) -> str:
     except (template.TemplateSyntaxError, AttributeError, TypeError, ValueError):
         rendered = ""
 
-    if force_text_preview:
+    if force_text_preview and not context.get("STATIC_EXPORT"):
+        # Read archived text only when the browser requests its preview, never
+        # while rendering the outer snapshot HTML (payloads may live on B2).
+        preview_url = _build_snapshot_preview_url(
+            str(_snapshot_id(result.snapshot)),
+            raw_output_path,
+            request=context.get("request"),
+            config=context.get("CONFIG"),
+            plugin=plugin,
+        )
+        return mark_safe(
+            f'<iframe src="{escape(preview_url)}" title="{escape(plugin)} preview" '
+            'loading="lazy" fetchpriority="low" sandbox="allow-scripts allow-same-origin" '
+            'style="border:0;pointer-events:none"></iframe>',
+        )
+
+    if force_text_preview and context.get("STATIC_EXPORT"):
         preview = _render_text_file_preview(result.snapshot_dir, raw_output_path, plugin, icon_html)
         if preview:
             return mark_safe(preview)
@@ -916,7 +932,11 @@ def plugin_full(context, result) -> str:
     output_url = _snapshot_url_for_context(context, result.snapshot, raw_output_path)
 
     try:
-        tpl = template.Template(template_str)
+        catalog = get_plugin_catalog()
+        tpl = template.Engine(
+            dirs=[catalog[plugin].path] if plugin in catalog else [],
+            libraries=template.Engine.get_default().libraries,
+        ).from_string(template_str)
         ctx = template.Context(
             {
                 "result": result,

archivebox/core/views.py

@@ -1,5 +1,7 @@
 __package__ = "archivebox.core"
 
+from archivebox.core.preview_util import PREVIEW_PLUGINS, preview_candidates
+
 import json
 import os
 import posixpath
@@ -23,6 +25,7 @@ from django.http import Http404, HttpRequest, HttpResponse, HttpResponseForbidde
 from django.shortcuts import redirect, render
 from django.utils.decorators import method_decorator
 from django.utils.html import format_html, format_html_join
+from django.utils.cache import patch_cache_control, patch_vary_headers
 from django.utils.safestring import mark_safe
 from django.views import View
 from django.views.decorators.csrf import csrf_exempt, csrf_protect
@@ -77,7 +80,7 @@ from archivebox.misc.util import (
     validate_url,
     without_fragment,
 )
-from archivebox.plugins.discovery import get_plugin_name, get_plugin_template
+from archivebox.plugins.discovery import get_plugin_catalog, get_plugin_name, get_plugin_template
 from archivebox.plugins.forms import get_plugin_config_binary_urls
 from archivebox.plugins.views import get_config_definition_link
 from archivebox.progressmonitor.views import live_progress_view
@@ -110,7 +113,10 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
     snapshots = Snapshot.objects.select_related("crawl", "crawl__created_by")
 
     if len(lookup) == 12 and "-" not in lookup:
-        return snapshots.filter(id__endswith=lookup).order_by("-created_at", "-downloaded_at").first()
+        # Resolve suffixes using the covering primary-key index before fetching
+        # wide snapshot rows and joining their crawl/user data.
+        matches = list(Snapshot.objects.filter(id__endswith=lookup).order_by().values_list("id", flat=True))
+        return snapshots.filter(pk__in=matches).order_by("-created_at", "-downloaded_at").first()
 
     try:
         return snapshots.get(pk=lookup)
@@ -327,6 +333,20 @@ class SnapshotView(View):
 
     @staticmethod
     def render_live_index(request, snapshot):
+        if request.GET.get("discover_outputs") == "1":
+            # Reuse the existing output discovery and card/badge classification.
+            request.archivebox_cache_policy = "private"
+            initial_context = snapshot.get_html_details_context(request=request)
+            known_paths = {output["path"] for output in initial_context["archiveresults"]}
+            context = snapshot.get_html_details_context(request=request, discover_files=True)
+            context["archiveresults"] = [output for output in context["archiveresults"] if output["path"] not in known_paths]
+            for output in context["archiveresults"]:
+                output["output_group"] = "other"
+                output["folder_path"] = str(Path(output["path"]).parent)
+            response = render(request, "core/snapshot_output_cards.html", context)
+            patch_cache_control(response, private=True, no_store=True)
+            return response
+        request.archivebox_cache_policy = "public" if snapshot.permissions == PERMISSIONS_PUBLIC else "private"
         return render(
             template_name="core/snapshot.html",
             request=request,
@@ -350,6 +370,7 @@ class SnapshotView(View):
             try:
                 try:
                     snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
+                    request.archivebox_cache_policy = "public" if snapshot.permissions == PERMISSIONS_PUBLIC else "private"
                     if not can_view_snapshot(request, snapshot):
                         return _private_snapshot_auth_redirect(request, snapshot, archivefile or "index.html")
                     canonical_base = snapshot.url_path
@@ -629,6 +650,7 @@ class SnapshotPathView(View):
                 status=404,
             )
 
+        request.archivebox_cache_policy = "public" if snapshot.permissions == PERMISSIONS_PUBLIC else "private"
         canonical_base = snapshot.url_path
         if date:
             requested_base = f"{username}/{date}/{domain or url or ''}"
@@ -721,7 +743,7 @@ def _plugin_full_preview_response(
     result: ArchiveResult | None,
 ) -> HttpResponse | None:
     """Render an explicit plugin full template as a trusted preview wrapper."""
-    if not request.GET.get("preview"):
+    if not request.GET.get("preview") or request.GET.get("raw") or request.GET.get("files"):
         return None
 
     path_parts = Path(rel_path).parts
@@ -729,11 +751,6 @@ def _plugin_full_preview_response(
     if not plugin:
         return None
 
-    # ReplayWeb.page needs plugin-owned WACZ inspection and service-worker
-    # context, so it remains the one narrow preview exception below.
-    if plugin == "archivewebpage" and archivewebpage_replay.is_replay_target(rel_path):
-        return None
-
     template_str = get_plugin_template(plugin, "full", fallback=False)
     if not template_str:
         return None
@@ -744,8 +761,9 @@ def _plugin_full_preview_response(
     if raw_query:
         output_url = f"{output_url}?{raw_query.urlencode()}"
 
+    catalog = get_plugin_catalog()
     rendered = (
-        template.Engine(debug=False)
+        template.Engine(debug=False, dirs=[catalog[plugin].path] if plugin in catalog else [])
         .from_string(template_str)
         .render(
             template.Context(
@@ -754,6 +772,7 @@ def _plugin_full_preview_response(
                     "snapshot": snapshot,
                     "output_path": output_url,
                     "output_path_raw": rel_path,
+                    "snapshot_details_url": build_web_url(snapshot.get_absolute_url(), request=request),
                     "plugin": plugin,
                     "preview_base": f"{request.path.rsplit('/', 1)[0]}/",
                 },
@@ -891,7 +910,7 @@ def _serve_responses_path(request, responses_root: Path, rel_path: str, show_ind
     return None
 
 
-def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""):
+def _build_snapshot_replay_response(request: HttpRequest, snapshot: Snapshot, path: str = ""):
     rel_path = path or ""
     request_config = get_request_config(
         request,
@@ -899,6 +918,7 @@ def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str =
     )
     request.archivebox_config = request_config
     request.archivebox_snapshot_url = snapshot.url
+    request.archivebox_cache_policy = "public" if snapshot.permissions == PERMISSIONS_PUBLIC else "private"
     snapshot._runtime_config = request_config
 
     if rel_path.startswith("replay/") or rel_path == "replay":
@@ -911,7 +931,7 @@ def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str =
         # /progress.json on through to the same view used everywhere else. The caller
         # passes snapshot_id explicitly in the query string — we don't read it from the
         # subdomain (this keeps the endpoint identical across all security modes).
-        return live_progress_view(request)
+        return live_progress_view(request, authorized_snapshot=snapshot)
 
     is_directory_request = bool(path) and path.endswith("/")
     show_indexes = bool(request.GET.get("files")) or (request_config.USES_SUBDOMAIN_ROUTING and is_directory_request)
@@ -954,6 +974,20 @@ def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str =
     raise Http404
 
 
+def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""):
+    response = _build_snapshot_replay_response(request, snapshot, path)
+    if path != "progress.json" and response.status_code in (200, 206, 304):
+        if not response.get("Cache-Control"):
+            policy = "public" if snapshot.permissions == PERMISSIONS_PUBLIC else "private"
+            response.headers["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
+        if snapshot.permissions != PERMISSIONS_PUBLIC:
+            patch_cache_control(response, private=True)
+    else:
+        response.headers["Cache-Control"] = "private, no-store"
+    patch_vary_headers(response, ("Cookie", "Authorization"))
+    return response
+
+
 def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str = ""):
     request_config = get_request_config(request, resolve_plugins=False)
     request.archivebox_config = request_config
@@ -977,6 +1011,8 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =
 
     root_match = (match[0], match[1]) if match else _latest_responses_root(snapshots, domain)
     responses_root = root_match[1] if root_match else None
+    if root_match:
+        request.archivebox_cache_policy = "public" if root_match[0].permissions == PERMISSIONS_PUBLIC else "private"
     if request_config.USES_SUBDOMAIN_ROUTING:
         snapshot = root_match[0] if root_match else (snapshots[0] if requested_root_index and snapshots else None)
         if snapshot:
@@ -1156,7 +1192,7 @@ class PublicIndexView(ListView):
         snapshots = list(context.get("object_list") or ())
         icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots}
         tag_names_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
-        preview_paths_by_snapshot: dict[str, list[tuple[int, int, str]]] = {str(snapshot.id): [] for snapshot in snapshots}
+        preview_results_by_snapshot = {str(snapshot.id): [] for snapshot in snapshots}
         favicon_paths_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
         progress_by_snapshot: dict[str, dict[str, int]] = {
             str(snapshot.id): {
@@ -1178,24 +1214,6 @@ class PublicIndexView(ListView):
             ):
                 tag_names_by_snapshot[str(snapshot_id)].append(tag_name)
 
-            preview_plugin_order = {
-                "screenshot": 0,
-                "chrome_extension_screenshot": 1,
-            }
-            preview_candidates = {
-                "screenshot": ("screenshot.png",),
-                "chrome_extension_screenshot": ("screenshot-1.png", "screenshot.png"),
-            }
-
-            def result_output_path(result: ArchiveResult, filename: str) -> str | None:
-                output_files = result.output_files or {}
-                file_info = output_files.get(filename)
-                if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
-                    return None
-                if file_info.get("root_relative"):
-                    return filename
-                return f"{result.plugin}/{filename}"
-
             for snapshot_id, plugin, status in (
                 ArchiveResult.objects.filter(
                     snapshot_id__in=icons_by_snapshot.keys(),
@@ -1222,23 +1240,16 @@ class PublicIndexView(ListView):
             for result in (
                 ArchiveResult.objects.filter(
                     snapshot_id__in=icons_by_snapshot.keys(),
-                    status=ArchiveResult.StatusChoices.SUCCEEDED,
-                    plugin__in=(*preview_candidates, "favicon"),
+                    plugin__in=PREVIEW_PLUGINS,
                 )
-                .only("snapshot_id", "plugin", "output_files")
+                .only("snapshot_id", "plugin", "status", "output_files")
                 .iterator(chunk_size=1000)
             ):
                 snapshot_key = str(result.snapshot_id)
-                if result.plugin in preview_candidates:
-                    plugin_rank = preview_plugin_order[result.plugin]
-                    for filename_rank, filename in enumerate(preview_candidates[result.plugin]):
-                        output_path = result_output_path(result, filename)
-                        if output_path:
-                            preview_paths_by_snapshot[snapshot_key].append((plugin_rank, filename_rank, output_path))
-                elif result.plugin == "favicon":
-                    output_path = result_output_path(result, "favicon.ico")
-                    if output_path:
-                        favicon_paths_by_snapshot[snapshot_key].append(output_path)
+                preview_results_by_snapshot[snapshot_key].append(result)
+                for candidate in preview_candidates([result]):
+                    if candidate["kind"] == "favicon":
+                        favicon_paths_by_snapshot[snapshot_key].append(candidate["path"])
 
         for snapshot in snapshots:
             snapshot._icons_compact = True
@@ -1246,9 +1257,7 @@ class PublicIndexView(ListView):
             snapshot._icons_progress_stats = progress_by_snapshot.get(str(snapshot.id), {})
             snapshot.num_outputs_cached = snapshot._icons_progress_stats.get("succeeded", 0)
             snapshot._tags_str_cached = ",".join(tag_names_by_snapshot.get(str(snapshot.id), []))
-            snapshot._public_preview_paths = [
-                output_path for _plugin_rank, _filename_rank, output_path in sorted(preview_paths_by_snapshot.get(str(snapshot.id), []))
-            ]
+            snapshot._preview_results = preview_results_by_snapshot.get(str(snapshot.id), [])
             snapshot._public_favicon_paths = favicon_paths_by_snapshot.get(str(snapshot.id), [])
             snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
         context["object_list"] = snapshots

archivebox/crawls/models.py

@@ -467,6 +467,8 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             "tags_str": self.tags_str,
             "label": self.label,
             "created_at": self.created_at.isoformat() if self.created_at else None,
+            "modified_at": self.modified_at.isoformat() if self.modified_at else None,
+            "created_by": self.created_by.username,
         }
 
     @staticmethod
@@ -844,24 +846,31 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         output_dir: Path | None = None,
         num_snapshots: int | None = None,
     ) -> str:
-        from abx_dl.limits import CrawlLimitState
+        from abx_dl.limits import parse_filesize_to_bytes
 
-        if output_dir is None:
-            output_dir = self.output_dir
         if config is None:
             from archivebox.config.common import get_config
 
-            config = get_config(crawl=self, include_machine=False).for_crawl_runtime(
-                crawl=self,
-                persona=self.resolve_persona(),
-                crawl_output_dir=output_dir,
-            )
+            config = get_config(crawl=self, include_machine=False)
 
-        limits_path = output_dir / ".abx-dl" / "limits.json"
-        if limits_path.exists():
-            stop_reason = CrawlLimitState.from_config(config).get_stop_reason()
-            if stop_reason:
-                return stop_reason
+        # The DB is the shared authority for workers and the UI. A second
+        # ledger beside the artifacts can disagree after cancellation/retry and
+        # would require filesystem access just to render a crawl's status.
+        # Recompute against the current limits so increasing a cap permits work.
+        timeout = int(self._config_value(config, "CRAWL_TIMEOUT", 0) or 0)
+        if timeout:
+            # Snapshot creation already supplies a durable crawl clock. Using
+            # that existing fact avoids another timestamp to keep in sync and
+            # survives cancellation, which deliberately removes unfinished
+            # ArchiveResult rows while preserving the Snapshot for retry.
+            first_snapshot_time = self.snapshot_set.aggregate(first=models.Min("created_at"))["first"]
+            if first_snapshot_time and (timezone.now() - first_snapshot_time).total_seconds() >= timeout:
+                return "crawl_timeout"
+        max_size = parse_filesize_to_bytes(self._config_value(config, "CRAWL_MAX_SIZE", 0) or 0)
+        if max_size:
+            total_size = self.snapshot_set.aggregate(total=models.Sum("output_size"))["total"] or 0
+            if total_size >= max_size:
+                return "crawl_max_size"
 
         max_urls = int(self._config_value(config, "CRAWL_MAX_URLS", 0) or 0)
         if num_snapshots is None:
@@ -1248,10 +1257,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             }
             if tag_names:
                 tag_names_by_url[snapshot.url] = tag_names
-            # Snapshot.save() handles model-level validation. The crawl symlink
-            # can still wait until after commit so SQLite does not hold a write
-            # lock while touching the filesystem.
-            transaction.on_commit(lambda snapshot=snapshot: snapshot.ensure_crawl_symlink())
 
         tag_names = {tag for tags in tag_names_by_url.values() for tag in tags}
         if tag_names:

archivebox/machine/detect.py

@@ -88,7 +88,7 @@ def get_vm_info():
         try:
             # Getting SMBIOS data from sysfs.
             # SMBIOS 2.8 present.
-            # argo-1        | 2024-10-01T10:40:51Z ERR  error="Incoming request ended abruptly: context canceled" connIndex=2 event=1 ingressRule=0 originService=http://archivebox:8000                                                                               │
+            # argo-1        | 2024-10-01T10:40:51Z ERR  error="Incoming request ended abruptly: context canceled" connIndex=2 event=1 ingressRule=0 originService=http://archivebox:5797                                                                               │
             # Handle 0x0100, DMI type 1, 27 bytes
             # System Information
             #         Manufacturer: DigitalOcean

archivebox/misc/hashing.py

@@ -1,260 +0,0 @@
-# Bootable hashing helpers (file content + mtime/size cache, mime detection).
-# MUST NOT import archivebox.config, archivebox.core, or Django — stdlib only.
-
-import hashlib
-import mimetypes
-from functools import lru_cache
-from pathlib import Path
-from collections.abc import Callable
-from datetime import datetime
-
-
-@lru_cache(maxsize=1024)
-def _cached_file_hash(filepath: str, size: int, mtime: float) -> str:
-    """Internal function to calculate file hash with cache key based on path, size and mtime."""
-    sha256_hash = hashlib.sha256()
-
-    with open(filepath, "rb") as f:
-        for chunk in iter(lambda: f.read(4096), b""):
-            sha256_hash.update(chunk)
-
-    return sha256_hash.hexdigest()
-
-
-@lru_cache(maxsize=10)
-def hash_file(file_path: Path, pwd: Path | None = None) -> str:
-    """Calculate SHA256 hash of a file with caching based on path, size and mtime."""
-    pwd = Path(pwd) if pwd else None
-    file_path = Path(file_path)
-    if not file_path.is_absolute():
-        file_path = pwd / file_path if pwd else file_path.absolute()
-
-    abs_path = file_path.resolve()
-    stat_info = abs_path.stat()
-
-    return _cached_file_hash(
-        str(abs_path),
-        stat_info.st_size,
-        stat_info.st_mtime,
-    )
-
-
-@lru_cache(maxsize=10)
-def get_dir_hashes(dir_path: Path, pwd: Path | None = None, filter_func: Callable | None = None, max_depth: int = -1) -> dict[str, str]:
-    """Calculate SHA256 hashes for all files and directories recursively."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    if not dir_path.is_dir():
-        raise ValueError(f"Not a directory: {dir_path}")
-    if max_depth < -1:
-        raise ValueError(f"max_depth must be >= -1, got {max_depth}")
-
-    # Get all files recursively
-    all_files = get_dir_entries(
-        dir_path,
-        pwd=pwd,
-        recursive=True,
-        include_files=True,
-        include_dirs=False,
-        filter_func=filter_func,
-    )
-
-    hashes: dict[str, str] = {}
-    hashable_summary = []
-
-    # Calculate hashes for all files
-    for subfile in all_files:
-        subfile_path = dir_path / subfile
-        sha256_hash = hash_file(subfile_path)
-        hashes[subfile] = sha256_hash
-        hashable_summary.append(f"{sha256_hash}  ./{subfile}")
-
-    # Calculate hashes for all directories
-    subdirs = get_dir_entries(
-        dir_path,
-        pwd=pwd,
-        recursive=True,
-        include_files=False,
-        include_dirs=True,
-        include_hidden=False,
-        filter_func=filter_func,
-        max_depth=max_depth,
-    )
-
-    for subdir in subdirs:
-        subdir_path = dir_path / subdir
-        subdir_hashes = get_dir_hashes(
-            subdir_path,
-            filter_func=filter_func,
-            max_depth=0,
-        )
-        hashes[subdir] = subdir_hashes["."]
-
-    # Filter results by max_depth
-    if max_depth >= 0:
-        hashes = {path: value for path, value in hashes.items() if len(Path(path).parts) <= max_depth + 1}
-
-    # Calculate root directory hash
-    hashable_summary.sort()
-    root_sha256 = hashlib.sha256("\n".join(hashable_summary).encode()).hexdigest()
-    hashes["."] = root_sha256
-
-    return hashes
-
-
-@lru_cache(maxsize=128)
-def get_dir_entries(
-    dir_path: Path,
-    pwd: Path | None = None,
-    recursive: bool = True,
-    include_files: bool = True,
-    include_dirs: bool = True,
-    include_hidden: bool = False,
-    filter_func: Callable | None = None,
-    max_depth: int = -1,
-) -> tuple[str, ...]:
-    """Get filtered list of directory entries."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    results = []
-
-    def process_path(path: Path, depth: int):
-        if not include_hidden and path.name.startswith("."):
-            return False
-        if max_depth >= 0 and depth > max_depth:
-            return False
-        if filter_func:
-            info = {
-                "abspath": str(path.absolute()),
-                "relpath": str(path.relative_to(dir_path)),
-            }
-            if not filter_func(info):
-                return False
-        return True
-
-    for path in dir_path.rglob("*") if recursive else dir_path.glob("*"):
-        current_depth = len(path.relative_to(dir_path).parts)
-
-        if path.is_file() and include_files and process_path(path, current_depth):
-            results.append(str(path.relative_to(dir_path)))
-        elif path.is_dir() and include_dirs and process_path(path, current_depth):
-            results.append(str(path.relative_to(dir_path)))
-
-        if not recursive:
-            break
-
-    return tuple(sorted(results))  # Make immutable for caching
-
-
-@lru_cache(maxsize=1024)
-def get_dir_sizes(dir_path: Path, pwd: Path | None = None, **kwargs) -> dict[str, int]:
-    """Calculate sizes for all files and directories recursively."""
-    sizes: dict[str, int] = {}
-    hashes = get_dir_hashes(dir_path, pwd=pwd, **kwargs)
-    dir_path = Path(dir_path)
-
-    for path_key in hashes:
-        full_path = dir_path / path_key
-        if full_path.is_file():
-            sizes[path_key] = full_path.stat().st_size
-        else:
-            total = 0
-            for file_path in full_path.rglob("*"):
-                if file_path.is_file() and not file_path.name.startswith("."):
-                    total += file_path.stat().st_size
-            sizes[path_key + "/"] = total
-
-    return sizes
-
-
-@lru_cache(maxsize=10)
-def get_dir_info(dir_path: Path, pwd: Path | None = None, filter_func: Callable | None = None, max_depth: int = -1) -> dict:
-    """Get detailed information about directory contents including hashes and sizes."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    hashes = get_dir_hashes(dir_path, pwd=pwd, filter_func=filter_func, max_depth=max_depth)
-    sizes = get_dir_sizes(str(dir_path), pwd=pwd, filter_func=filter_func, max_depth=max_depth)
-
-    num_total_subpaths = sum(1 for name in hashes if name != ".")
-    details = {}
-
-    for filename, sha256_hash in sorted(hashes.items()):
-        abs_path = (dir_path / filename).resolve()
-        stat_info = abs_path.stat()
-        num_subpaths = sum(1 for p in hashes if p.startswith(filename + "/"))
-        is_dir = abs_path.is_dir()
-        if is_dir:
-            mime_type = "inode/directory"
-            basename = abs_path.name
-            extension = ""
-            num_bytes = sizes[filename + "/"]
-            if filename == ".":
-                num_subpaths = num_total_subpaths
-            else:
-                filename += "/"
-                num_subpaths = num_subpaths
-        else:  # is_file
-            num_subpaths = None
-            mime_type = mimetypes.guess_type(str(abs_path))[0]
-            extension = abs_path.suffix
-            basename = abs_path.name.rsplit(extension, 1)[0]
-            num_bytes = sizes[filename]
-
-        details[filename] = {
-            "basename": basename,
-            "mime_type": mime_type,
-            "extension": extension,
-            "num_subpaths": num_subpaths,
-            "num_bytes": num_bytes,
-            "hash_sha256": sha256_hash,
-            "created_at": datetime.fromtimestamp(stat_info.st_ctime).isoformat(),
-            "modified_at": datetime.fromtimestamp(stat_info.st_mtime).isoformat(),
-        }
-
-        if filter_func and not filter_func(details[filename]):
-            del details[filename]
-
-    return details
-
-
-if __name__ == "__main__":
-    import json
-
-    dir_info = get_dir_info(Path("."), max_depth=6)
-    with open(".hashes.json", "w") as f:
-        json.dump(dir_info, f, indent=4)
-    print("Wrote .hashes.json")
-
-# Example output:
-# {
-#     ".": {
-#         "basename": "misc",
-#         "mime_type": "inode/directory",
-#         "extension": "",
-#         "num_subpaths": 25,
-#         "num_bytes": 214677,
-#         "hash_sha256": "addfacf88b2ff6b564846415fb7b21dcb7e63ee4e911bc0aec255ee354958530",
-#         "created_at": "2024-12-04T00:08:38.537449",
-#         "modified_at": "2024-12-04T00:08:38.537449"
-#     },
-#     "__init__.py": {
-#         "basename": "__init__",
-#         "mime_type": "text/x-python",
-#         "extension": ".py",
-#         "num_subpaths": null,
-#         "num_bytes": 32,
-#         "hash_sha256": "b0e5e7ff17db3b60535cf664282787767c336e3e203a43e21b6326c6fe457551",
-#         "created_at": "2024-10-08T00:51:41.001359",
-#         "modified_at": "2024-10-08T00:51:41.001359"
-#     },
-#     ...
-# }

archivebox/misc/jsonl.py

@@ -32,7 +32,6 @@ import sys
 import json
 from typing import Any, TextIO
 from collections.abc import Iterable, Iterator
-from pathlib import Path
 
 
 # Type constants for JSONL records
@@ -120,19 +119,6 @@ def read_stdin(stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
             yield record
 
 
-def read_file(path: Path) -> Iterator[dict[str, Any]]:
-    """
-    Read JSONL or plain URLs from a file.
-
-    Yields parsed records as dicts.
-    """
-    with open(path) as f:
-        for line in f:
-            record = parse_line(line)
-            if record:
-                yield record
-
-
 def read_args_or_stdin(args: Iterable[str], stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
     """
     Read from CLI arguments if provided, otherwise from stdin.

archivebox/misc/serve_static.py

@@ -14,6 +14,7 @@ import time
 import zipfile
 from collections.abc import Callable
 from datetime import UTC, datetime
+from html.parser import HTMLParser
 from pathlib import Path
 from urllib.parse import quote, urlencode, urljoin
 
@@ -129,7 +130,7 @@ def _resolve_archive_path(document_root: str | Path, rel_path: str) -> tuple[Pat
 
 def _cache_policy(config=None, **config_kwargs) -> str:
     config = config or get_config(resolve_plugins=False, **config_kwargs)
-    return "private" if config.PERMISSIONS == "private" else "public"
+    return "public" if config.PERMISSIONS == "public" else "private"
 
 
 def _format_direntry_timestamp(stat_result: os.stat_result) -> str:
@@ -527,6 +528,26 @@ def _index_responses_paths_for_html_images(
 
     best_matches: dict[str, str] = {}
     image_matches: set[str] = set()
+    # Optional responses plugin file contract: index.jsonl records contain method,
+    # url, status, and a path relative to responses/. No plugin code is imported.
+    # Legacy captures without an index still use the filename lookup below.
+    # The index retains complete URLs even when response filenames are shortened.
+    index = responses_root / "index.jsonl"
+    if index.is_file():
+        with index.open(errors="replace") as records:
+            for line in records:
+                try:
+                    record = json.loads(line)
+                    encoded = _encoded_responses_image_url(record.get("url", ""), None)
+                    if encoded not in encoded_urls or record.get("method") != "GET" or record.get("status") != 200:
+                        continue
+                    candidate = (responses_root / record.get("path", "")).resolve()
+                    if not candidate.is_relative_to(responses_root.resolve()) or not candidate.is_file():
+                        continue
+                    best_matches[encoded] = posixpath.relpath(candidate, start=snapshot_root / posixpath.dirname(html_rel_path))
+                    image_matches.add(encoded)
+                except (ValueError, TypeError, AttributeError):
+                    continue
     image_suffixes = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"}
     for candidate in responses_root.rglob("*"):
         if not candidate.is_file():
@@ -561,10 +582,30 @@ def _rewrite_html_image_sources_to_responses(
         if image_url not in encoded_urls_by_src:
             encoded_urls_by_src[image_url] = _encoded_responses_image_url(image_url, page_url)
 
+    # Optional dom plugin artifact: dom/output.html. Only image src and
+    # data-canonical-src attributes are read; no scripts or plugin code execute.
+    # Absence of this artifact simply disables proxy-to-original aliases.
+    aliases: dict[str, str] = {}
+
+    class ImageAliases(HTMLParser):
+        def handle_starttag(self, tag, attrs):
+            attrs = dict(attrs)
+            original = attrs.get("data-canonical-src")
+            proxy = attrs.get("src")
+            if tag == "img" and original and proxy:
+                original_key = _encoded_responses_image_url(original, page_url)
+                proxy_key = _encoded_responses_image_url(proxy, page_url)
+                if original_key and proxy_key:
+                    aliases[original_key] = proxy_key
+
+    dom = snapshot_root / "dom" / "output.html"
+    if dom.is_file() and dom.stat().st_size <= 10 * 1024 * 1024:
+        ImageAliases().feed(dom.read_text(errors="replace"))
+    wanted = {encoded for encoded in encoded_urls_by_src.values() if encoded}
     response_paths = _index_responses_paths_for_html_images(
         snapshot_root,
         html_rel_path,
-        {encoded_url for encoded_url in encoded_urls_by_src.values() if encoded_url},
+        wanted | {aliases[key] for key in wanted if key in aliases},
     )
     rewrites = 0
 
@@ -572,7 +613,7 @@ def _rewrite_html_image_sources_to_responses(
         nonlocal rewrites
         image_url = html.unescape(match.group(2))
         encoded_url = encoded_urls_by_src.get(image_url)
-        local_path = response_paths.get(encoded_url or "")
+        local_path = response_paths.get(encoded_url or "") or response_paths.get(aliases.get(encoded_url or "", ""))
         if not local_path:
             return match.group(0)
         rewrites += 1
@@ -589,12 +630,17 @@ def _rewrite_html_image_sources_for_request(
 ) -> tuple[str, int]:
     if not document_root:
         return html_text, 0
-    return _rewrite_html_image_sources_to_responses(
-        html_text,
-        document_root,
-        html_rel_path,
-        request.__dict__.get("archivebox_snapshot_url"),
-    )
+    try:
+        return _rewrite_html_image_sources_to_responses(
+            html_text,
+            document_root,
+            html_rel_path,
+            request.__dict__.get("archivebox_snapshot_url"),
+        )
+    except (OSError, ValueError, TypeError, RecursionError):
+        # Saved plugin artifacts are optional and may be incomplete, malformed,
+        # or removed concurrently. Image enrichment must not break file replay.
+        return html_text, 0
 
 
 def _apply_transformed_html_preview_style(html_text: str) -> str:
@@ -605,16 +651,22 @@ def _apply_transformed_html_preview_style(html_text: str) -> str:
     return f"{TRANSFORMED_HTML_PREVIEW_STYLE}\n{html_text}"
 
 
-def _set_transformed_response_headers(response, fullpath: Path, statobj: os.stat_result, encoding: str | None, config) -> None:
+def _set_transformed_response_headers(
+    response,
+    fullpath: Path,
+    statobj: os.stat_result,
+    encoding: str | None,
+    cache_policy: str,
+) -> None:
     response.headers["Last-Modified"] = http_date(statobj.st_mtime)
-    response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
+    response.headers["Cache-Control"] = f"{cache_policy}, max-age=60, stale-while-revalidate=300"
     response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"'
     if encoding:
         response.headers["Content-Encoding"] = encoding
 
 
 def _render_markdown_fallback(text: str) -> str:
-    if _markdown is not None and not HTML_TAG_RE.search(text):
+    if _markdown is not None:
         try:
             return _markdown(
                 text,
@@ -858,19 +910,22 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
     config = request.__dict__.get("archivebox_config")
     if config is None:
         config = get_config(resolve_plugins=False)
+    cache_policy = getattr(request, "archivebox_cache_policy", None) or _cache_policy(config=config)
     fullpath, path = _resolve_archive_path(document_root, path)
     if os.access(fullpath, os.R_OK) and fullpath.is_dir():
         if request.GET.get("download") == "zip" and show_indexes:
-            return _build_directory_zip_response(
+            response = _build_directory_zip_response(
                 fullpath,
                 path,
                 is_archive_replay=is_archive_replay,
                 use_async_stream=_is_asgi_request(request),
                 config=config,
             )
+            response.headers["Cache-Control"] = f"{cache_policy}, max-age=60, stale-while-revalidate=300"
+            return response
         if show_indexes:
             response = _render_directory_index(request, path, fullpath)
-            response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
+            response.headers["Cache-Control"] = f"{cache_policy}, max-age=60, stale-while-revalidate=300"
             response.headers["Last-Modified"] = http_date(fullpath.stat().st_mtime)
             return _apply_archive_replay_headers(
                 response,
@@ -899,7 +954,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             if etag in inm_list or etag.strip('"') in [i.strip('"') for i in inm_list]:
                 not_modified = HttpResponseNotModified()
                 not_modified.headers["ETag"] = etag
-                not_modified.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable"
+                not_modified.headers["Cache-Control"] = f"{cache_policy}, max-age=31536000, immutable"
                 not_modified.headers["Last-Modified"] = http_date(statobj.st_mtime)
                 return _apply_archive_replay_headers(
                     not_modified,
@@ -910,7 +965,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 )
 
     content_type, encoding = mimetypes.guess_type(str(fullpath))
-    preserve_plain_text = fullpath.suffix.lower() in {".log", ".sh"}
+    preserve_plain_text = fullpath.suffix.lower() in {".log", ".sh", ".jsonl"}
     if preserve_plain_text:
         content_type = "text/plain"
     content_type = content_type or "application/octet-stream"
@@ -927,7 +982,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
     preview_as_text_html = (
         bool(request.GET.get("preview"))
         and is_text_like
-        and not content_type.startswith("text/html")
+        and (bool(request.GET.get("raw")) or not content_type.startswith("text/html"))
         and not content_type.startswith("image/svg+xml")
     )
     preview_as_image_html = (
@@ -940,8 +995,10 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
         request.META.get("HTTP_IF_MODIFIED_SINCE"),
         statobj.st_mtime,
     ):
+        not_modified = HttpResponseNotModified()
+        not_modified.headers["Cache-Control"] = f"{cache_policy}, max-age=60, stale-while-revalidate=300"
         return _apply_archive_replay_headers(
-            HttpResponseNotModified(),
+            not_modified,
             fullpath=fullpath,
             content_type=content_type,
             is_archive_replay=is_archive_replay,
@@ -956,7 +1013,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 decoded = fullpath.read_text(encoding="utf-8", errors="replace")
                 wrapped = _render_text_preview_document(decoded, fullpath.name)
                 response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
-                _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
+                _set_transformed_response_headers(response, fullpath, statobj, encoding, cache_policy)
                 return _apply_archive_replay_headers(
                     response,
                     fullpath=fullpath,
@@ -976,7 +1033,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 raw_image_url = f"{raw_image_url}?{urlencode(list(preview_query.lists()), doseq=True)}"
             wrapped = _render_image_preview_document(raw_image_url, fullpath.name)
             response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
-            _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
+            _set_transformed_response_headers(response, fullpath, statobj, encoding, cache_policy)
             return _apply_archive_replay_headers(
                 response,
                 fullpath=fullpath,
@@ -1002,9 +1059,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 last_modified=http_date(statobj.st_mtime),
                 etag=etag or "",
                 cache_control=(
-                    f"{_cache_policy(config=config)}, max-age=31536000, immutable"
-                    if etag
-                    else f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
+                    f"{cache_policy}, max-age=31536000, immutable" if etag else f"{cache_policy}, max-age=60, stale-while-revalidate=300"
                 ),
                 content_encoding=encoding or "",
             )
@@ -1023,7 +1078,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
 
     # Heuristic fix: some archived HTML outputs are stored with HTML-escaped markup
     # or markdown sources. If so, render sensibly.
-    if not preserve_plain_text and content_type.startswith(("text/plain", "text/html")):
+    if not request.GET.get("raw") and not preserve_plain_text and content_type.startswith(("text/plain", "text/html", "text/markdown")):
         try:
             max_unescape_size = 10 * 1024 * 1024  # 10MB cap to avoid heavy memory use
             if statobj.st_size <= max_unescape_size:
@@ -1037,12 +1092,12 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 if content_type.startswith("text/html") and document_root:
                     rewritten_html, rewritten_count = _rewrite_html_image_sources_for_request(request, decoded, document_root, rel_path)
                 markdown_candidate = _extract_markdown_candidate(decoded)
-                if _looks_like_markdown(markdown_candidate):
+                if content_type.startswith("text/markdown") or _looks_like_markdown(markdown_candidate):
                     wrapped = _render_markdown_document(markdown_candidate)
                     wrapped, _rewrite_count = _rewrite_html_image_sources_for_request(request, wrapped, document_root, rel_path)
                     wrapped = _apply_transformed_html_preview_style(wrapped)
                     response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
-                    _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
+                    _set_transformed_response_headers(response, fullpath, statobj, encoding, cache_policy)
                     return _apply_archive_replay_headers(
                         response,
                         fullpath=fullpath,
@@ -1053,7 +1108,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 if rewritten_count:
                     rewritten_html = _apply_transformed_html_preview_style(rewritten_html)
                     response = HttpResponse(rewritten_html, content_type=content_type)
-                    _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
+                    _set_transformed_response_headers(response, fullpath, statobj, encoding, cache_policy)
                     return _apply_archive_replay_headers(
                         response,
                         fullpath=fullpath,
@@ -1064,7 +1119,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 if escaped_count and escaped_count > tag_count * 2:
                     decoded = _apply_transformed_html_preview_style(decoded)
                     response = HttpResponse(decoded, content_type=content_type)
-                    _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
+                    _set_transformed_response_headers(response, fullpath, statobj, encoding, cache_policy)
                     return _apply_archive_replay_headers(
                         response,
                         fullpath=fullpath,
@@ -1084,13 +1139,13 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
     response.headers["Last-Modified"] = http_date(statobj.st_mtime)
     if etag:
         response.headers["ETag"] = etag
-        response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable"
+        response.headers["Cache-Control"] = f"{cache_policy}, max-age=31536000, immutable"
     else:
-        response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
+        response.headers["Cache-Control"] = f"{cache_policy}, max-age=60, stale-while-revalidate=300"
     if is_text_like:
         response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"'
     if content_type.startswith("image/"):
-        response.headers["Cache-Control"] = "public, max-age=604800, immutable"
+        response.headers["Cache-Control"] = f"{cache_policy}, max-age=604800, immutable"
 
     # handle byte-range requests by serving chunk of file
     if stat.S_ISREG(statobj.st_mode):

archivebox/misc/toml_util.py

@@ -2,94 +2,11 @@ from typing import Any, cast
 from collections.abc import Callable
 
 import json
-import ast
-import inspect
 import toml
 import re
-import configparser
 
 from pathlib import Path, PosixPath
 
-from pydantic.json_schema import GenerateJsonSchema
-from pydantic_core import to_jsonable_python
-
-JSONValue = str | bool | int | None | list["JSONValue"]
-
-TOML_HEADER = "# Converted from INI to TOML format: https://toml.io/en/\n\n"
-
-
-def load_ini_value(val: str) -> JSONValue:
-    """Convert lax INI values into strict TOML-compliant (JSON) values"""
-    if val.lower() in ("true", "yes", "1"):
-        return True
-    if val.lower() in ("false", "no", "0"):
-        return False
-    if val.isdigit():
-        return int(val)
-
-    try:
-        return ast.literal_eval(val)
-    except Exception:
-        pass
-
-    try:
-        return json.loads(val)
-    except Exception:
-        pass
-
-    return val
-
-
-def convert(ini_str: str) -> str:
-    """Convert a string of INI config into its TOML equivalent (warning: strips comments)"""
-
-    config = configparser.ConfigParser()
-    setattr(config, "optionxform", str)  # capitalize key names
-    config.read_string(ini_str)
-
-    # Initialize an empty dictionary to store the TOML representation
-    toml_dict = {}
-
-    # Iterate over each section in the INI configuration
-    for section in config.sections():
-        toml_dict[section] = {}
-
-        # Iterate over each key-value pair in the section
-        for key, value in config.items(section):
-            parsed_value = load_ini_value(value)
-
-            # Convert the parsed value to its TOML-compatible JSON representation
-            toml_dict[section.upper()][key.upper()] = json.dumps(parsed_value)
-
-    # Build the TOML string
-    toml_str = TOML_HEADER
-    for section, items in toml_dict.items():
-        toml_str += f"[{section}]\n"
-        for key, value in items.items():
-            toml_str += f"{key} = {value}\n"
-        toml_str += "\n"
-
-    return toml_str.strip()
-
-
-class JSONSchemaWithLambdas(GenerateJsonSchema):
-    """
-    Encode lambda functions in default values properly.
-    Usage:
-    >>> json.dumps(value, encoder=JSONSchemaWithLambdas())
-    """
-
-    def encode_default(self, dft: Any) -> Any:
-        config = self._config
-        if isinstance(dft, Callable):
-            return "{{lambda " + inspect.getsource(dft).split("=lambda ")[-1].strip()[:-1] + "}}"
-        return to_jsonable_python(
-            dft,
-            timedelta_mode=config.ser_json_timedelta,
-            bytes_mode=config.ser_json_bytes,
-            serialize_unknown=True,
-        )
-
 
 def better_toml_dump_str(val: Any) -> str:
     try:

archivebox/personas/admin.py

@@ -27,7 +27,7 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
             "Persona",
             {
                 "fields": ("name", "created_by", "permissions"),
-                "classes": ("card", "persona-card-primary"),
+                "classes": ("card", "wide", "persona-card-primary"),
             },
         ),
         (
@@ -42,7 +42,7 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
                     "import_extract_cookies",
                     "import_capture_storage",
                 ),
-                "classes": ("card", "wide"),
+                "classes": ("card", "wide", "persona-card-import"),
             },
         ),
         (
@@ -63,7 +63,6 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
                 "classes": ("card", "persona-card-timestamps"),
             },
         ),
-        add_fieldsets[1],
         add_fieldsets[2],
         (
             "Artifacts",
@@ -136,7 +135,7 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
         return super().get_form(request, obj=obj, change=change, **kwargs)
 
     def render_change_form(self, request, context, add=False, change=False, form_url="", obj=None):
-        context["detected_profile_count"] = len(discover_local_browser_profiles())
+        context["detected_profile_count"] = len(discover_local_browser_profiles()) if add else 0
         return super().render_change_form(request, context, add=add, change=change, form_url=form_url, obj=obj)
 
     def save_model(self, request, obj, form, change):

archivebox/personas/forms.py

@@ -55,7 +55,7 @@ class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm):
         label="Autodiscovered profiles",
         widget=forms.RadioSelect,
         choices=(),
-        help_text="Detected from local Chrome, Chromium, Brave, and Edge profile roots.",
+        help_text="Detected from local Chrome, Chrome Beta, Chromium, Brave, and Edge profile roots.",
     )
     import_source = forms.CharField(
         required=False,

archivebox/personas/importers.py

@@ -18,6 +18,7 @@ import subprocess
 import sys
 import tempfile
 from dataclasses import dataclass, field
+from functools import partial
 from pathlib import Path
 from typing import TYPE_CHECKING
 from urllib.parse import urlparse
@@ -31,6 +32,7 @@ if TYPE_CHECKING:
 
 BROWSER_LABELS = {
     "chrome": "Google Chrome",
+    "chrome-beta": "Google Chrome Beta",
     "chromium": "Chromium",
     "brave": "Brave",
     "edge": "Microsoft Edge",
@@ -216,24 +218,29 @@ class PersonaImportResult:
         return self.profile_copied or self.cookies_imported or self.storage_captured or self.user_agent_imported
 
 
-def get_chrome_user_data_dir() -> Path | None:
+def get_chrome_user_data_dir(*, beta: bool = False) -> Path | None:
     """Get the default Chrome user data directory for the current platform."""
     system = platform.system()
     home = Path.home()
+    app_name = "Chrome Beta" if beta else "Chrome"
 
     if system == "Darwin":
         candidates = [
-            home / "Library" / "Application Support" / "Google" / "Chrome",
+            home / "Library" / "Application Support" / "Google" / app_name,
         ]
     elif system == "Linux":
-        candidates = [
-            home / ".config" / "google-chrome",
-            home / ".config" / "chrome",
-        ]
+        candidates = (
+            [home / ".config" / "google-chrome-beta"]
+            if beta
+            else [
+                home / ".config" / "google-chrome",
+                home / ".config" / "chrome",
+            ]
+        )
     elif system == "Windows":
         local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
         candidates = [
-            local_app_data / "Google" / "Chrome" / "User Data",
+            local_app_data / "Google" / app_name / "User Data",
         ]
     else:
         candidates = []
@@ -320,6 +327,7 @@ def get_edge_user_data_dir() -> Path | None:
 
 BROWSER_PROFILE_FINDERS = {
     "chrome": get_chrome_user_data_dir,
+    "chrome-beta": partial(get_chrome_user_data_dir, beta=True),
     "chromium": get_chromium_user_data_dir,
     "brave": get_brave_user_data_dir,
     "edge": get_edge_user_data_dir,
@@ -354,6 +362,12 @@ def validate_persona_name(name: str) -> tuple[bool, str]:
 
 
 def discover_local_browser_profiles() -> list[PersonaImportSource]:
+    # The macOS companion exports with the host Keychain before creating personas.
+    # Raw mounts alone cannot decrypt macOS cookies in the Linux container.
+    if host_profiles := os.environ.get("ARCHIVEBOX_HOST_BROWSER_PROFILES"):
+        host_templates = discover_persona_template_profiles(Path(host_profiles))
+        host_names = {source.source_name for source in host_templates}
+        return host_templates + [source for source in discover_persona_template_profiles() if source.source_name not in host_names]
     discovered: list[PersonaImportSource] = []
 
     for browser, finder in BROWSER_PROFILE_FINDERS.items():
@@ -430,6 +444,11 @@ def discover_persona_template_profiles(personas_dir: Path | None = None) -> list
 
 def resolve_browser_import_source(browser: str, profile_dir: str | None = None) -> PersonaImportSource:
     browser = browser.lower().strip()
+    if host_profiles := os.environ.get("ARCHIVEBOX_HOST_BROWSER_PROFILES"):
+        label = BROWSER_LABELS.get(browser, browser)
+        exported = Path(host_profiles) / f"{label} - {profile_dir or 'Default'}"
+        if exported.is_dir():
+            return resolve_custom_import_source(str(exported))
     if browser not in BROWSER_PROFILE_FINDERS:
         supported = ", ".join(BROWSER_PROFILE_FINDERS)
         raise ValueError(f"Unknown browser: {browser}. Supported browsers: {supported}")
@@ -547,6 +566,7 @@ def resolve_source_browser_binary(source: PersonaImportSource) -> str:
         return str(Path(source.browser_binary).expanduser().resolve())
     names = {
         "chrome": ("Google Chrome", "google-chrome"),
+        "chrome-beta": ("Google Chrome Beta", "google-chrome-beta"),
         "chromium": ("Chromium", "chromium"),
         "brave": ("Brave Browser", "brave-browser"),
         "edge": ("Microsoft Edge", "microsoft-edge"),
@@ -561,6 +581,7 @@ def resolve_source_browser_binary(source: PersonaImportSource) -> str:
     elif platform.system() == "Windows":
         relative = {
             "chrome": "Google/Chrome/Application/chrome.exe",
+            "chrome-beta": "Google/Chrome Beta/Application/chrome.exe",
             "edge": "Microsoft/Edge/Application/msedge.exe",
             "brave": "BraveSoftware/Brave-Browser/Application/brave.exe",
             "chromium": "Chromium/Application/chrome.exe",

archivebox/plugins/discovery.py

@@ -55,10 +55,6 @@ def get_plugins() -> list[str]:
     return sorted(get_plugin_catalog())
 
 
-def get_plugin_models():
-    return get_plugin_catalog().plugins
-
-
 def get_plugin_name(plugin: str) -> str:
     """
     Get the base plugin name without numeric prefix.

archivebox/plugins/forms.py

@@ -213,7 +213,7 @@ class PluginConfigFormMixin:
                     "enabled_config_key": enabled_config_key,
                     "description": str(schema.get("description") or "").strip(),
                     "source_url": f"https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/{plugin_name}",
-                    "docs_url": f"https://archivebox.github.io/abx-plugins/#{plugin_name}",
+                    "docs_url": f"https://plugins.archivebox.io/#{plugin_name}",
                     "required_plugins": [str(item) for item in schema.get("required_plugins") or []],
                     "required_binary_links": _build_required_binary_links(
                         schema.get("required_binaries") or [],

archivebox/plugins/output_groups.py

@@ -0,0 +1,95 @@
+"""Snapshot output taxonomy. Display preference is independent of hook order."""
+
+from pathlib import PurePosixPath
+from typing import Any
+
+from archivebox.plugins.discovery import get_plugin_name
+
+# Ordered tuples are preference lists; unordered groups sort by total output bytes.
+OUTPUT_GROUPS = (
+    ("html", "HTML", ("archivewebpage", "singlefile", "chrome_mhtml", "wget", "dom")),
+    ("raster", "Raster", ("screenshot", "pdf")),
+    ("article_text", "Article text", ("readability", "defuddle", "mercury", "trafilatura", "htmltotext")),
+    ("embedded_media", "Embedded media", ()),
+    ("metadata", "Metadata", ()),
+    ("other", "Other files", ()),
+)
+UNORDERED_OUTPUTS = {
+    "embedded_media": {"responses", "ytdlp", "yt-dlp", "youtube-dl", "gallerydl", "forumdl", "git", "media", "liteparse"},
+    "metadata": {
+        "dns",
+        "headers",
+        "seo",
+        "accessibility",
+        "redirects",
+        "consolelog",
+        "sslcerts",
+        "ssl",
+        "title",
+        "favicon",
+        "hashes",
+        "chrome",
+        "parse_html_urls",
+        "parse_txt_urls",
+        "parse_dom_outlinks",
+        "parse_jsonl_urls",
+        "parse_netscape_urls",
+        "parse_rss_urls",
+        "tlsnotary",
+        "opentimestamps",
+    },
+}
+
+
+def _responses_html_card(outputs: list[dict[str, Any]]) -> dict[str, Any] | None:
+    # Presentation-only exception to the usual 1:1 plugin:card rule. Responses
+    # owns both this HTML view and its media gallery; responses_html is only a
+    # unique card/anchor ID, never a plugin or a synthetic ArchiveResult.
+    # Refactor this exception if plugins gain support for multiple cards.
+    for output in outputs:
+        if output["name"] != "responses" or not (result := output.get("result")):
+            continue
+        path = str(output.get("path") or "")
+        if not path.startswith("responses/") or ".." in PurePosixPath(path).parts:
+            continue
+        files = result.output_file_map()
+        info = files.get(path) or files.get(path.removeprefix("responses/")) or {}
+        mime = str(info.get("mimetype") or "").split(";")[0].lower()
+        if not info or result._coerce_output_file_size(info.get("size")) <= 0:
+            continue
+        if mime not in {"text/html", "application/xhtml+xml"} and PurePosixPath(path).suffix.lower() not in {".html", ".htm"}:
+            continue
+        return {
+            "name": "responses_html",
+            "path": path,
+            "folder_path": "responses",
+            "ts": output.get("ts"),
+            "size": 0,  # Bytes are already counted by the real Responses card.
+            "result": None,
+            "direct_preview_path": f"{path}?card=responses_html",
+        }
+    return None
+
+
+def order_snapshot_outputs(outputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Annotate and order available cards; unknown plugins stay visible in Other files."""
+    group_order = {group_id: index for index, (group_id, _, _) in enumerate(OUTPUT_GROUPS)}
+    preferences = {}
+    for group_id, _, plugins in OUTPUT_GROUPS:
+        for rank, plugin in enumerate(plugins):
+            preferences[plugin] = (group_id, rank)
+    for group_id, plugins in UNORDERED_OUTPUTS.items():
+        for plugin in plugins:
+            preferences[plugin] = (group_id, None)
+
+    if card := _responses_html_card(outputs):
+        outputs = [*outputs, card]
+    preferences["responses_html"] = ("html", len(OUTPUT_GROUPS[0][2]))
+
+    def sort_key(output):
+        plugin = get_plugin_name(output["name"])
+        group_id, rank = preferences.get(plugin, ("other", None))
+        output["output_group"] = group_id
+        return (group_order[group_id], rank if rank is not None else -int(output.get("size") or 0), plugin)
+
+    return sorted(outputs, key=sort_key)

archivebox/plugins/views.py

@@ -17,7 +17,7 @@ from archivebox.config.common import get_live_config_url
 from archivebox.config.views import get_environment_binary_url, is_superuser
 from archivebox.plugins.discovery import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, discover_plugin_configs, iter_plugin_dirs
 
-ABX_PLUGINS_DOCS_BASE_URL = "https://archivebox.github.io/abx-plugins/"
+ABX_PLUGINS_DOCS_BASE_URL = "https://plugins.archivebox.io/"
 ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/"
 LIVE_PLUGIN_BASE_URL = "/admin/environment/plugins/"
 

archivebox/progressmonitor/views.py

@@ -43,7 +43,7 @@ def _live_progress_plugin_names() -> tuple[frozenset[str], frozenset[str]]:
     return download_plugin_names, indexing_plugin_names
 
 
-def live_progress_view(request):
+def live_progress_view(request, *, authorized_snapshot=None):
     """Simple JSON endpoint for live progress status - used by admin progress monitor."""
     try:
         from archivebox.core.models import ArchiveResult, Snapshot
@@ -63,7 +63,10 @@ def live_progress_view(request):
             except (TypeError, ValueError):
                 return JsonResponse({"error": "Invalid snapshot_id"}, status=400)
             scoped_snapshot = Snapshot.objects.filter(id=snapshot_id_filter).select_related("crawl").first()
-            if scoped_snapshot is None or not can_view_snapshot(request, scoped_snapshot):
+            authorized_snapshot_id = str(getattr(authorized_snapshot, "id", "")).replace("-", "")
+            if scoped_snapshot is None or (
+                authorized_snapshot_id != str(scoped_snapshot.id).replace("-", "") and not can_view_snapshot(request, scoped_snapshot)
+            ):
                 return JsonResponse({"error": "Permission denied"}, status=403)
         elif crawl_id_filter:
             # Crawl-only scope still requires staff: there's no per-crawl ACL helper,
@@ -221,6 +224,8 @@ def live_progress_view(request):
 
         api_base = get_api_base_url(request=request, config=request_config) if scoped_snapshot is not None else ""
 
+        running_screencasts = {}
+
         def screencast_frame_url(crawl_id: str, crawl_dir: Path) -> str:
             frame_path = crawl_dir / "chrome_screencast" / "latest.jpg"
             try:
@@ -229,7 +234,8 @@ def live_progress_view(request):
                 return ""
             if frame_stat.st_size <= 0:
                 return ""
-            if now.timestamp() - frame_stat.st_mtime > 15:
+            started_at = running_screencasts.get(crawl_id)
+            if started_at is None or frame_stat.st_mtime < started_at.timestamp():
                 return ""
             rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}"
             return f"{api_base}{rel}" if api_base else rel
@@ -475,7 +481,10 @@ def live_progress_view(request):
         detailed_snapshot_ids = [snapshot["id"] for snapshot in snapshots if snapshot["status"] != Snapshot.StatusChoices.QUEUED]
         process_value_fields = ("id", "process_type", "status", "pwd", "cmd", "pid", "exit_code", "started_at", "ended_at", "modified_at")
         if active_crawl_ids or displayed_snapshot_ids:
+            relevant_process_ids = tuple(dict.fromkeys([*active_crawl_ids, *displayed_snapshot_ids]))
+            relevant_process_path_pattern = "|".join(relevant_process_ids)
             process_scope = Process.objects.filter(
+                pwd__regex=relevant_process_path_pattern,
                 machine_id=machine_id,
                 process_type__in=[
                     Process.TypeChoices.HOOK,
@@ -552,6 +561,10 @@ def live_progress_view(request):
                 snapshot_id = str(matched_snapshot["id"])
             running_worker_ids.add(str(proc["id"]))
             _plugin, _label, phase, _hook_name = process_label(proc["cmd"])
+            if _plugin == "chrome_screencast" and proc["started_at"] and Process(**proc, machine_id=machine_id).is_running:
+                # CDP sends frames only when the page changes. Keep a still
+                # frame visible for its live capture, never for a previous run.
+                running_screencasts[crawl_id] = max(proc["started_at"], running_screencasts.get(crawl_id, proc["started_at"]))
             if crawl_id and proc["pid"]:
                 crawl_process_pids.setdefault(crawl_id, proc["pid"])
             if phase == "snapshot" and snapshot_id and proc["pid"]:

archivebox/services/archive_result_service.py

@@ -141,6 +141,23 @@ def _save_archiveresult_event_to_db(
     if snapshot is None:
         return
 
+    if event.status == "cancelled":
+        # Stopping a capture tells the user nothing about whether it would have
+        # succeeded. Remove this attempt instead of showing a failure card or
+        # adding it to "retry failed". The next scheduled run creates a new row.
+        results = ArchiveResult.objects.filter(snapshot=snapshot, plugin=event.plugin, hook_name=event.hook_name)
+        if process_started is not None:
+            # A late completion must not erase a newer retry of the same hook.
+            results = results.filter(start_ts=parse_event_datetime(process_started.start_ts))
+        # Ordinary deletion invokes output-directory cleanup. Cancellation is
+        # only withdrawing the DB fact: retain partial files and sibling hooks'
+        # shared output: they may be the only copy we ever capture. A rerun's
+        # hook owns replacing files without deleting the old copy first (see
+        # abx-plugins README, Rules). Retain Process/logs to explain the stop.
+        results._raw_delete(results.db)
+        ArchiveResult.refresh_snapshot_output_sizes([snapshot.id])
+        return
+
     with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.plugin_dir"):
         plugin_dir = (
             Path(process_started.output_dir)
@@ -224,7 +241,15 @@ def _save_archiveresult_event_to_db(
 
 
 def mark_archiveresult_started(event: ProcessStartedEvent, *, snapshot_id: str, process_id: str) -> None:
-    """Project a running abx-dl hook after its OS process is persisted."""
+    """Project a running abx-dl hook after its OS process is persisted.
+
+    This DB state records process execution; the scheduler separately waits for
+    stdout readiness before advancing past a background hook. Neither boundary
+    proves that an archive was produced. Final status comes from the owning
+    hook's ArchiveResultEvent, including corrections after an abnormal exit.
+    Rows are per snapshot/plugin/hook: a recorder's start hook cannot declare
+    that its separate export hook succeeded.
+    """
     from archivebox.core.models import ArchiveResult, Snapshot
 
     started_at = parse_event_datetime(event.start_ts)

archivebox/services/runner.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import contextvars
+import json
 import os
 import signal
 import shutil
@@ -12,7 +13,7 @@ from contextlib import nullcontext
 from datetime import timedelta
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Any
+from typing import Any, Literal
 
 from asgiref.sync import sync_to_async
 from django.core.exceptions import ValidationError
@@ -26,6 +27,7 @@ from abx_dl.events import (
     CrawlCleanupEvent,
     CrawlCompletedEvent,
     CrawlEvent,
+    CrawlPauseEvent,
     CrawlSetupEvent,
     CrawlStartEvent,
     InstallEvent,
@@ -36,7 +38,6 @@ from abx_dl.events import (
     SnapshotEvent,
     slow_warning_timeout,
 )
-from abx_dl.limits import CrawlLimitState
 from abx_dl.catalog import PluginCatalog
 from abx_dl.config import GlobalConfig, RuntimeConfig
 from abx_dl.models import Snapshot as AbxSnapshot
@@ -198,7 +199,12 @@ class CrawlRunner:
         self.crawl = crawl
         self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0)
         self.catalog = get_plugin_catalog()
-        HookProcessService(self.bus, emit_jsonl=False, interactive_tty=interactive_interrupts)
+        self.process_service = HookProcessService(
+            self.bus,
+            emit_jsonl=False,
+            interactive_tty=interactive_interrupts,
+            interrupted_hook_prompt=self._prompt_in_foreground if interactive_interrupts else None,
+        )
         register_sonic_daemon_event_handler(self.bus)
         PersistedProcessService(self.bus)
         ArchiveBoxBinaryService(self.bus)
@@ -209,6 +215,9 @@ class CrawlRunner:
         self.process_discovered_snapshots_inline = process_discovered_snapshots_inline
         self.show_progress = show_progress
         self.interactive_interrupts = interactive_interrupts
+        self._interrupt_choice: asyncio.Future[Literal["abort", "retry", "skip"]] | None = None
+        self._user_aborted = False
+        self.bus.on(CrawlAbortEvent, self.on_CrawlAbortEvent)
         self.config_overrides = dict(config_overrides or {})
 
         self.snapshot_service = SnapshotService(
@@ -231,9 +240,11 @@ class CrawlRunner:
         self.primary_url = ""
         self.crawl_output_dir = ""
         self._live_stream = None
+        self._live_ui: LiveBusUI | None = None
         self.root_crawl_event_id: str | None = None
         self.root_crawl_start_event_id: str | None = None
         self._run_task: asyncio.Task[None] | None = None
+        self._run_started = False
         self._skip_wait_until_idle = False
         # This is intentionally a synchronous OS-signal side channel, not bus
         # state. During SIGINT/SIGTERM/SIGHUP, asyncio.run() may already be
@@ -242,21 +253,115 @@ class CrawlRunner:
         self._signal_abort_requested = False
         self._last_lease_heartbeat_at = 0.0
 
-    def _request_abort_from_signal(self, _sig: signal.Signals) -> None:
-        if os.environ.get("ARCHIVEBOX_RUNNER_DAEMON") == "1":
-            # The daemon runner is owned by supervisord, not by the interactive
-            # CLI foreground flow. A direct signal to this child should be short
-            # and unambiguous: exit non-zero immediately so supervisord restarts
-            # the runner, while the parent server and supervisord stay alive.
-            os._exit(128 + int(_sig))
-        already_requested = self._signal_abort_requested
+    async def on_CrawlAbortEvent(self, event: CrawlAbortEvent) -> None:
+        self._signal_abort_requested = True
+        # An abort can arrive while the prompt transport is being established,
+        # before another SIGINT sees _interrupt_choice. Release that waiter too;
+        # shutdown must never depend on a subsequent keypress or IPC response.
+        if self._interrupt_choice is not None and not self._interrupt_choice.done():
+            self._interrupt_choice.set_result("abort")
+        if event.user_initiated:
+            self._user_aborted = True
+            # Exit 130 alone cannot distinguish a user's abort from a worker
+            # displaced by takeover. Tell the owning add command explicitly
+            # so it exits instead of reclaiming and restarting this crawl.
+            if os.environ.get("SUPERVISOR_PROCESS_NAME"):
+                sys.__stderr__.write("ARCHIVEBOX_CRAWL_ABORTED\n")
+                sys.__stderr__.flush()
+
+    def _interrupt_hook(self) -> None:
+        if self._user_aborted:
+            # An explicit abort is already draining hook groups with normal
+            # grace periods. A further Ctrl+C requests immediate force exit.
+            # This service owns only this runner's hook groups and children;
+            # a server's supervisor or another foreground command is outside
+            # the boundary. Tell a supervising add parent this was user intent
+            # before bypassing the worker's normal finalization path.
+            if os.environ.get("SUPERVISOR_PROCESS_NAME"):
+                os.write(sys.__stderr__.fileno(), b"ARCHIVEBOX_CRAWL_ABORTED\n")
+            os.write(sys.__stderr__.fileno(), b"\n[!] Forcing aborted crawl to exit now.\n")
+            self.process_service.force_kill_owned_processes()
+            os._exit(130)
+        if self._run_task is None:
+            # The crawl is already draining/finalizing; there is no live bus on
+            # which to reopen a prompt. Remember the user's abort and finish the
+            # cleanup that is already in progress. In particular do not hand a
+            # second signal to Python's default handler in this teardown window.
+            self._user_aborted = True
+            if os.environ.get("SUPERVISOR_PROCESS_NAME"):
+                os.write(sys.__stderr__.fileno(), b"ARCHIVEBOX_CRAWL_ABORTED\n")
+            stream = self._live_stream if self._live_stream is not None and not self._live_stream.closed else sys.__stderr__
+            os.write(stream.fileno(), b"\nAborting crawl - finishing shutdown... Ctrl+C again to force exit.\n")
+            return
+        if self._interrupt_choice is not None:
+            self._choose_interrupt_action("abort")
+            return
+
+        async def dispatch() -> None:
+            # ArchiveBox transports terminal intent; abx-dl alone decides pause
+            # versus abort. No local counter may compete with its controller.
+            await self.bus.emit(CrawlPauseEvent()).now()
+
+        asyncio.get_running_loop().create_task(dispatch())
+
+    async def _prompt_in_foreground(self, hook_name: str) -> Literal["abort", "retry", "skip"]:
+        rendered = self._live_ui.show_interrupt_prompt(hook_name) if self._live_ui is not None else False
+        if sys.stdin.isatty():
+            # Direct run is itself the outer terminal owner. Keep its asyncio
+            # loop alive so background completions render above the prompt. No
+            # supervisor parent exists here to consume an IPC prompt request.
+            choice = (
+                await HookProcessService.read_interrupt_choice(
+                    hook_name,
+                    render=not rendered,
+                    is_active=lambda: not self._signal_abort_requested,
+                    on_abort=lambda: setattr(self, "_user_aborted", True),
+                )
+                or "abort"
+            )
+            if choice == "abort":
+                # Also cover an inactive prompt that defaulted to abort. A
+                # typed answer is recorded at the keypress by on_abort above.
+                self._user_aborted = True
+            return choice
+        # A supervised worker has log pipes instead of terminal input. Its add
+        # parent owns the same shared reader and returns the answer via signals.
+        self._interrupt_choice = asyncio.get_running_loop().create_future()
+        sys.__stderr__.write("ARCHIVEBOX_HOOK_INTERRUPTED " + json.dumps({"hook_name": hook_name, "rendered": rendered}) + "\n")
+        sys.__stderr__.flush()
+        try:
+            return await self._interrupt_choice
+        finally:
+            self._interrupt_choice = None
+
+    def _choose_interrupt_action(self, choice: Literal["abort", "retry", "skip"]) -> None:
+        if self._interrupt_choice is not None and not self._interrupt_choice.done():
+            if choice == "abort":
+                self._user_aborted = True
+            self._interrupt_choice.set_result(choice)
+
+    def _request_abort_from_signal(self, sig: signal.Signals) -> None:
+        # Daemons also need cooperative cleanup. os._exit used to leave hook
+        # process groups orphaned and Process rows running forever. A daemon's
+        # nonzero exit still lets supervisord restart it, AFTER finally blocks
+        # stop descendants and finalize the ledger. SIGINT uses the same shared
+        # controller as other runners, with interactive_tty=False (whole abort).
+        # Takeover/parent loss stops this execution owner, not the user's crawl
+        # intent. Do not set _user_aborted or emit ARCHIVEBOX_CRAWL_ABORTED here:
+        # the outer add may need to wait for the new owner and reclaim afterward.
+        # Cancellation intentionally bypasses the interactive pause gate; a
+        # waiting prompt must never keep a displaced worker or its hooks alive.
         self._signal_abort_requested = True
         self._skip_wait_until_idle = True
+        if not self._run_started and os.environ.get("ARCHIVEBOX_RUNNER_DAEMON") == "1":
+            # A signal before run() starts has no async task or hooks to drain.
+            # It must still be an unexpected supervisor exit, not a clean idle exit.
+            raise SystemExit(128 + sig.value)
         # The foreground signal handler runs while the event loop may be in the
         # middle of shutdown. Flip cheap in-memory flags here and let normal
-        # finally blocks do cleanup; only cancel the runner task immediately for
-        # non-interactive commands or for a second interrupt escalation.
-        if (not self.interactive_interrupts or already_requested) and self._run_task is not None and not self._run_task.done():
+        # finally blocks do cleanup. Interactive SIGINT uses _interrupt_hook;
+        # actual shutdown signals must cancel promptly in either mode.
+        if self._run_task is not None and not self._run_task.done():
             self._run_task.cancel()
 
     async def crawl_is_cancelled(self) -> bool:
@@ -284,7 +389,7 @@ class CrawlRunner:
             await asyncio.sleep(poll_interval)
             if not await self.crawl_is_cancelled():
                 continue
-            abort_event = parent_event.emit(CrawlAbortEvent())
+            abort_event = parent_event.emit(CrawlAbortEvent(user_initiated=False))
             await _run_event_now(abort_event, abort_event.event_timeout)
             return
 
@@ -299,25 +404,36 @@ class CrawlRunner:
         root_snapshot_id: str | None = None
         bus_destroyed = False
         run_state_loaded = False
-        try:
-            first_signal_message = (
-                "\n[🛑] Got {signal_name}, aborting the active hook...\n"
-                if self.interactive_interrupts
-                else "\n[🛑] Got {signal_name}, stopping gracefully...\n"
-            )
-            self._run_task = asyncio.current_task()
-            # Do not raise KeyboardInterrupt directly from an OS signal while
-            # the asyncio loop is active. Python can inject it into whichever
-            # task is currently running, which produces noisy "Task exception
-            # was never retrieved" logs from unrelated abxbus housekeeping
-            # tasks. _request_abort_from_signal() cancels the runner task
-            # cooperatively instead; repeated signals still hard-exit in the
-            # shared foreground signal handler.
-            with foreground_shutdown_signals(
-                first_signal_message=first_signal_message,
-                on_signal=self._request_abort_from_signal,
-                raise_on_first_signal=False,
-            ):
+        self._run_task = asyncio.current_task()
+        self._run_started = True
+        # Do not raise KeyboardInterrupt directly from an OS signal while
+        # the asyncio loop is active. Python can inject it into whichever
+        # task is currently running, which produces noisy "Task exception
+        # was never retrieved" logs from unrelated abxbus housekeeping
+        # tasks. _request_abort_from_signal() cancels the runner task
+        # cooperatively instead; repeated signals still hard-exit in the
+        # shared foreground signal handler.
+        # Keep signal ownership through bus destruction AND durable finalization.
+        # Restoring the outer/default handler before those awaits leaves a real
+        # race: a rapid second Ctrl+C can kill Python during teardown, producing
+        # supervisor exit -1 / shell 255 instead of an acknowledged user abort.
+        with foreground_shutdown_signals(
+            first_signal_message="\n[🛑] Got {signal_name}, stopping crawl worker...\n",
+            on_signal=self._request_abort_from_signal,
+            interrupt_handlers={
+                signal.SIGINT: self._interrupt_hook,
+                **(
+                    {
+                        signal.SIGUSR1: lambda: self._choose_interrupt_action("skip"),
+                        signal.SIGUSR2: lambda: self._choose_interrupt_action("retry"),
+                    }
+                    if self.interactive_interrupts
+                    else {}
+                ),
+            },
+            raise_on_first_signal=False,
+        ):
+            try:
                 if await sync_to_async(defer_crawl_for_resources, thread_sensitive=True)(self.crawl):
                     return
                 # Startup and its first snapshot are one admission. Browser
@@ -330,6 +446,7 @@ class CrawlRunner:
                 self.max_concurrent_snapshots = max_concurrent_snapshots
                 self.snapshot_semaphore = asyncio.Semaphore(max_concurrent_snapshots)
                 live_ui = self._create_live_ui()
+                self._live_ui = live_ui
                 with live_ui if live_ui is not None else nullcontext():
                     try:
                         if snapshot_ids:
@@ -339,26 +456,30 @@ class CrawlRunner:
                         self._run_task = None
                         await self.stop_snapshot_tasks()
                         try:
+                            await self.process_service.stop_background_hooks()
                             await self.bus.wait_until_idle(timeout=1.0 if self._skip_wait_until_idle else 30.0)
                         except TimeoutError:
                             pass
                         finally:
                             await self.bus.destroy(clear=False)
                             bus_destroyed = True
-        finally:
-            if not bus_destroyed:
-                self._run_task = None
-                await self.stop_snapshot_tasks()
-                await self.bus.destroy(clear=False)
-            if self._live_stream is not None:
-                try:
-                    self._live_stream.close()
-                except Exception:
-                    pass
-                self._live_stream = None
-            if run_state_loaded:
-                await sync_to_async(project_abxpkg_derived_cache_to_db, thread_sensitive=True)(self.base_config.get("ABXPKG_LIB_DIR"))
-                await sync_to_async(self.finalize_run_state, thread_sensitive=True)()
+            finally:
+                if not bus_destroyed:
+                    self._run_task = None
+                    await self.stop_snapshot_tasks()
+                    await self.process_service.stop_background_hooks()
+                    await self.bus.destroy(clear=False)
+                if self._live_stream is not None:
+                    try:
+                        self._live_stream.close()
+                    except Exception:
+                        pass
+                    self._live_stream = None
+                if run_state_loaded:
+                    await sync_to_async(project_abxpkg_derived_cache_to_db, thread_sensitive=True)(self.base_config.get("ABXPKG_LIB_DIR"))
+                    await sync_to_async(self.finalize_run_state, thread_sensitive=True)()
+            if self._user_aborted:
+                raise KeyboardInterrupt
 
     async def enqueue_snapshot(self, snapshot_id: str, crawl_start_event: CrawlStartEvent | None = None) -> None:
         if self._resource_deferred:
@@ -697,10 +818,12 @@ class CrawlRunner:
         stderr_is_tty = sys.stderr.isatty()
         interactive_tty = stdout_is_tty or stderr_is_tty
         stream = sys.stderr if stderr_is_tty or not stdout_is_tty else sys.stdout
-        if interactive_tty and os.path.exists("/dev/tty"):
+        progress_tty = os.environ.get("ARCHIVEBOX_PROGRESS_TTY")
+        if progress_tty or (interactive_tty and os.path.exists("/dev/tty")):
             try:
-                self._live_stream = open("/dev/tty", "w", buffering=1, encoding=stream.encoding or "utf-8")
+                self._live_stream = open(progress_tty or "/dev/tty", "w", buffering=1, encoding=stream.encoding or "utf-8")
                 stream = self._live_stream
+                interactive_tty = stream.isatty()
             except OSError:
                 self._live_stream = None
         try:
@@ -1033,7 +1156,9 @@ class CrawlRunner:
             snapshot_selected_plugins = (
                 self.requested_plugins or snapshot["retry_plugins"] or snapshot["selected_plugins"] or self.selected_plugins
             )
-            if snapshot["depth"] > 0 and CrawlLimitState.from_config(snapshot["config"]).get_stop_reason() in (
+            # All remaining URLs share the crawl budget, including explicitly
+            # submitted roots. abx-dl only enforces the current snapshot's size.
+            if await sync_to_async(snapshot["_snapshot"].crawl.limit_stop_reason, thread_sensitive=True)(config=config) in (
                 "crawl_max_size",
                 "crawl_timeout",
             ):
@@ -1643,11 +1768,19 @@ def run_install(*, plugin_names: list[str] | None = None) -> None:
     asyncio.run(_run_install(plugin_names=plugin_names))
 
 
-def _first_due_id(queryset):
-    return queryset.order_by("retry_at", "created_at").values_list("id", flat=True).first()
+def _first_due_id(queryset, *, newest_first: bool = False):
+    ordering = ("-retry_at", "-created_at") if newest_first else ("retry_at", "created_at")
+    return queryset.order_by(*ordering).values_list("id", flat=True).first()
 
 
-def _run_due_crawl_status(status: str, *, crawl_id: str | None, lock_seconds: int, interactive_interrupts: bool) -> bool:
+def _run_due_crawl_status(
+    status: str,
+    *,
+    crawl_id: str | None,
+    lock_seconds: int,
+    interactive_interrupts: bool,
+    newest_first: bool = False,
+) -> bool:
     from archivebox.crawls.models import Crawl
 
     due_crawls = Crawl.objects.filter(
@@ -1656,7 +1789,7 @@ def _run_due_crawl_status(status: str, *, crawl_id: str | None, lock_seconds: in
     )
     if crawl_id:
         due_crawls = due_crawls.filter(id=crawl_id)
-    due_crawl_id = _first_due_id(due_crawls)
+    due_crawl_id = _first_due_id(due_crawls, newest_first=newest_first)
     if due_crawl_id is None:
         return False
     due_crawl = Crawl.objects.filter(id=due_crawl_id).first()
@@ -1670,8 +1803,15 @@ def _run_due_crawl_status(status: str, *, crawl_id: str | None, lock_seconds: in
     return True
 
 
-def _run_due_snapshot_query(queryset, *, lock_seconds: int, interactive_interrupts: bool, runtime_config) -> bool:
-    due_snapshot_id = _first_due_id(queryset)
+def _run_due_snapshot_query(
+    queryset,
+    *,
+    lock_seconds: int,
+    interactive_interrupts: bool,
+    runtime_config,
+    newest_first: bool = False,
+) -> bool:
+    due_snapshot_id = _first_due_id(queryset, newest_first=newest_first)
     return _run_due_snapshot_id(
         due_snapshot_id,
         lock_seconds=lock_seconds,
@@ -1779,20 +1919,31 @@ def run_pending_crawls(
             )
             if crawl_id:
                 active_snapshots = active_snapshots.filter(crawl_id=crawl_id)
+            # Admit new submissions between captures instead of making them
+            # wait behind every overdue child of an old, resumed crawl.
             if _run_due_snapshot_query(
-                active_snapshots,
+                active_snapshots.filter(crawl__status=Crawl.StatusChoices.QUEUED),
                 lock_seconds=60,
                 interactive_interrupts=interactive_interrupts,
                 runtime_config=runtime_config,
+                newest_first=True,
             ):
                 continue
 
-        if not maintenance_only:
             if _run_due_crawl_status(
                 Crawl.StatusChoices.QUEUED,
                 crawl_id=crawl_id,
                 lock_seconds=crawl_claim_lock_seconds,
                 interactive_interrupts=interactive_interrupts,
+                newest_first=True,
+            ):
+                continue
+
+            if _run_due_snapshot_query(
+                active_snapshots,
+                lock_seconds=60,
+                interactive_interrupts=interactive_interrupts,
+                runtime_config=runtime_config,
             ):
                 continue
 

archivebox/services/snapshot_service.py

@@ -6,7 +6,6 @@ from pathlib import Path
 from asgiref.sync import sync_to_async
 from django.utils import timezone
 from abx_dl.events import SnapshotCompletedEvent, SnapshotEvent
-from abx_dl.limits import CrawlLimitState
 from abx_dl.services.base import BaseService
 
 
@@ -34,7 +33,7 @@ def project_discovered_snapshots(snapshot_id: str) -> list:
         crawl_output_dir=crawl.output_dir,
         snapshot_output_dir=snapshot.output_dir,
     )
-    if CrawlLimitState.from_config(config).get_stop_reason() in ("crawl_max_size", "crawl_timeout"):
+    if crawl.limit_stop_reason(config=config) in ("crawl_max_size", "crawl_timeout"):
         return []
     return crawl.create_discovered_snapshots(snapshot, discovered_urls, depth=snapshot.depth + 1)
 
@@ -114,12 +113,7 @@ def finalize_completed_snapshot(
 def _crawl_limit_stop_reason(crawl) -> str:
     from archivebox.config.common import get_config
 
-    config_model = get_config(crawl=crawl)
-    config = config_model.for_crawl_runtime(
-        crawl=crawl,
-        persona=crawl.resolve_persona(),
-    )
-    return CrawlLimitState.from_config(config).get_stop_reason()
+    return crawl.limit_stop_reason(config=get_config(crawl=crawl))
 
 
 class SnapshotService(BaseService):

archivebox/templates/admin/base.html

@@ -1339,20 +1339,36 @@
                 display: inline-block;
             }
 
-            .action-rearchive-select,
-            .action-permissions-select {
+            /* The native select receives the actual click, including in WKWebView. */
+            body.change-list #changelist .actions .action-rearchive-select,
+            body.change-list #changelist .actions .action-permissions-select {
                 position: absolute;
-                left: 0;
-                bottom: 0;
-                width: 1px;
-                height: 1px;
+                inset: 0;
+                width: 100%;
+                height: 100%;
                 opacity: 0;
-                pointer-events: none;
-                border: 0;
-                padding: 0;
+                cursor: pointer;
+                z-index: 1;
                 margin: 0;
             }
 
+            body.change-list #changelist .action-rearchive-wrapper > .button,
+            body.change-list #changelist .action-permissions-wrapper > .button {
+                padding: 8px 16px !important;
+                width: max-content;
+                min-width: max-content;
+                flex: 0 0 auto;
+                gap: 6px;
+                white-space: nowrap;
+            }
+
+            .action-rearchive-wrapper:focus-within,
+            .action-permissions-wrapper:focus-within {
+                outline: 2px solid #2563eb;
+                outline-offset: 2px;
+                border-radius: 6px;
+            }
+
             /* Object count */
             .actions .action-counter {
                 color: #64748b;
@@ -1960,22 +1976,10 @@
                             .addClass('action-rearchive-select')
                             .attr('aria-label', 'Re-Archive')
                             .appendTo(wrapper)
-                        $('<button>')
-                            .attr('type', 'button')
+                        $('<span>')
+                            .attr('aria-hidden', 'true')
                             .addClass('button')
                             .html('Re-Archive <span aria-hidden="true">▾</span>')
-                            .click(function (e) {
-                                e.preventDefault()
-                                e.stopPropagation()
-                                const el = select[0]
-                                if (typeof el.showPicker === 'function') {
-                                    try { el.showPicker() } catch (err) { el.focus(); el.click() }
-                                } else {
-                                    el.focus()
-                                    el.click()
-                                }
-                                return false
-                            })
                             .appendTo(wrapper)
                         $('<option></option>').attr('value', '').attr('hidden', true).appendTo(select)
                         rearchiveActions.forEach(function (action_value) {
@@ -2021,23 +2025,11 @@
                             .addClass('action-permissions-select')
                             .attr('aria-label', 'Permissions')
                             .appendTo(wrapper)
-                        $('<button>')
-                            .attr('type', 'button')
+                        $('<span>')
+                            .attr('aria-hidden', 'true')
                             .attr('name', action_type)
                             .addClass('button')
                             .text(this.text)
-                            .click(function (e) {
-                                e.preventDefault()
-                                e.stopPropagation()
-                                const el = select[0]
-                                if (typeof el.showPicker === 'function') {
-                                    try { el.showPicker() } catch (err) { el.focus(); el.click() }
-                                } else {
-                                    el.focus()
-                                    el.click()
-                                }
-                                return false
-                            })
                             .appendTo(wrapper)
                         $('<option></option>').attr('value', '').attr('hidden', true).appendTo(select)
                         ;[

archivebox/templates/admin/personas/persona/change_form.html

@@ -38,8 +38,8 @@
         border-bottom: 1px dashed #64748b;
     }
 
-    .field-import_mode ul,
-    .field-import_discovered_profile ul {
+    .abx-import-mode,
+    .abx-profile-picker {
         margin: 0;
         padding: 0;
         list-style: none;
@@ -47,20 +47,26 @@
         gap: 12px;
     }
 
-    .field-import_mode ul {
-        grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
+    .abx-import-mode {
+        grid-template-columns: repeat(auto-fit, minmax(min(220px, 100%), 1fr));
     }
 
-    .field-import_discovered_profile ul {
-        grid-template-columns: repeat(auto-fit, minmax(260px, 1fr));
-        max-height: 460px;
-        overflow: auto;
-        padding-right: 4px;
+    .abx-profile-picker {
+        grid-template-columns: repeat(auto-fit, minmax(min(260px, 100%), 1fr));
     }
 
-    .field-import_mode li,
-    .field-import_discovered_profile li {
+    .abx-import-mode > div,
+    .abx-profile-picker > div {
         margin: 0;
+        min-width: 0;
+    }
+
+    /* Import rows grow with their contents instead of sharing a fixed flex
+       height. Avoid overflow-x:hidden turning overflow-y:visible into auto. */
+    #content-main #persona_form .persona-card-import > .form-row,
+    #content-main #persona_form .persona-card-import fieldset > div {
+        flex: none;
+        overflow: visible;
     }
 
     .field-import_mode label,
@@ -68,7 +74,6 @@
         display: flex;
         gap: 12px;
         align-items: flex-start;
-        min-height: 100%;
         padding: 14px 16px;
         border-radius: 14px;
         border: 1px solid #dbe4ee;
@@ -167,9 +172,14 @@
         color: #991b1b;
     }
 
-    body.model-persona.change-form #content-main form fieldset.persona-card-primary {
-        flex: 1 1 520px !important;
-        max-width: calc(66.66% - 10px) !important;
+    #content-main #persona_form .field-name .flex-container {
+        max-width: 100% !important;
+    }
+
+    #persona_form .field-name input {
+        width: 100%;
+        max-width: none;
+        box-sizing: border-box;
     }
 
     body.model-persona.change-form #content-main form fieldset.persona-card-timestamps {
@@ -221,6 +231,7 @@
 
 {% block extrahead %}
 {{ block.super }}
+{% if add %}
 <script>
 document.addEventListener('DOMContentLoaded', function () {
     const modeInputs = Array.from(document.querySelectorAll('input[name="import_mode"]'));
@@ -246,9 +257,11 @@ document.addEventListener('DOMContentLoaded', function () {
     }
 });
 </script>
+{% endif %}
 {% endblock %}
 
 {% block form_top %}
+{% if add %}
 <section class="persona-import-hero">
     <div>
         <h2>Create a new Persona (Authentication + Configuration Profile)</h2>
@@ -257,16 +270,22 @@ document.addEventListener('DOMContentLoaded', function () {
             or by pasting in a CDP url from
             <code class="persona-import-copy" data-copy-text="chrome://inspect#remote-debugging" title="Copy chrome://inspect#remote-debugging">chrome://inspect#remote-debugging</code> (e.g. <code>http://127.0.0.1:9222</code>).
         </p>
+        <p>
+            You can also import cookies into this persona from another browser using the
+            <a href="https://extension.archivebox.io/" target="_blank" rel="noopener noreferrer">ArchiveBox browser extension</a>.
+            Create the persona here, then select it in the extension's settings to sync cookies from that browser.
+        </p>
     </div>
 </section>
+{% endif %}
 {{ block.super }}
 {% endblock %}
 
 {% block after_field_sets %}
 {{ block.super }}
-<section id="plugin-config" class="module aligned persona-plugin-config">
+<fieldset id="plugin-config" class="module aligned wide persona-plugin-config">
     <h2>Plugin Config</h2>
     <p>These typed controls update the same Persona config JSON shown above. Shared config keys stay synced across plugin sections.</p>
     {% include "plugins/plugin_config_grid.html" with plugin_groups=adminform.form.plugin_groups %}
-</section>
+</fieldset>
 {% endblock %}

archivebox/templates/admin/snapshots_grid.html

@@ -147,11 +147,21 @@
   }
   .cards .card .card-meta {
       display: grid;
-      grid-template-columns: 18px minmax(0, 1fr) auto;
+      grid-template-columns: minmax(0, 1fr) auto minmax(0, 1fr);
       gap: 8px;
       align-items: center;
       text-align: center;
   }
+  .cards .card .card-meta .action-select {
+      justify-self: start;
+  }
+  .cards .card .card-meta .card-date {
+      justify-self: center;
+      max-width: 100%;
+  }
+  .cards .card .card-meta .card-size {
+      justify-self: end;
+  }
   .cards .card .card-meta .card-date,
   .cards .card .card-meta .card-size {
       min-width: 0;
@@ -249,22 +259,7 @@
           <code class="card-url" title="{{obj.url}}">{{obj.url}}</code>
           <div class="card-media">
             <a href="{% snapshot_base_url obj %}/index.html" class="card-thumbnail">
-              {% snapshot_archiveresult_url obj 'screenshot' 'screenshot.png' as screenshot_url %}
-              {% snapshot_archiveresult_url obj 'chrome_extension_screenshot' 'screenshot-1.png' as extension_screenshot_1_url %}
-              {% snapshot_archiveresult_url obj 'chrome_extension_screenshot' 'screenshot.png' as extension_screenshot_url %}
-              {% snapshot_archiveresult_url obj 'favicon' 'favicon.ico' as favicon_url %}
-              {% if screenshot_url or extension_screenshot_1_url or extension_screenshot_url or favicon_url %}
-                <img
-                  src="{% firstof screenshot_url extension_screenshot_1_url extension_screenshot_url favicon_url %}"
-                  alt="{{obj.title|default:'Not yet archived...'}}"
-                  data-fallbacks="{{ extension_screenshot_1_url }},{{ extension_screenshot_url }},{{ favicon_url }}"
-                  onerror="const fallbacks=(this.dataset.fallbacks || '').split(',').filter(Boolean); if (fallbacks.length) { this.dataset.fallbacks=fallbacks.slice(1).join(','); this.src=fallbacks[0]; } else { this.onerror=null; this.closest('.card-thumbnail').classList.add('missing'); this.remove(); }"
-                />
-              {% elif obj.status == 'started' %}
-                <img src="{% static 'spinner.gif' %}" class="loading-preview" alt="Archiving in progress"/>
-              {% else %}
-                <span class="missing-preview">No preview captured</span>
-              {% endif %}
+              {% snapshot_thumbnail obj admin=True width="100%" height="245px" %}
             </a>
               {% if obj.tags_str %}
                 <div class="card-tags">

archivebox/templates/core/add.html

@@ -37,6 +37,7 @@
                     Check the server log or the <a href="/admin/core/archiveresult/?o=-1">Log</a> page for detailed progress...
                 </center>
             </div>
+            <input id="url-file-input" type="file" multiple hidden>
             <form id="add-form" method="POST" class="p-form">{% csrf_token %}
                 <center>
                     <h1>Create a new Crawl</h1>
@@ -69,6 +70,11 @@
                                     <pre id="url-highlight-layer" class="url-highlight-layer" aria-hidden="true"></pre>
                                     {{ form.url }}
                                 </div>
+                                <div class="url-upload-controls">
+                                    <button type="button" id="url-upload-button"><span aria-hidden="true">↥</span> Upload</button>
+                                    <span>or drag &amp; drop in files containing URLs</span>
+                                </div>
+                                <div id="url-upload-status" class="help-text" role="status"></div>
                             </div>
                             <aside class="detected-urls-panel" aria-live="polite">
                                 <div class="detected-urls-header">
@@ -287,7 +293,8 @@
                                 <code>daily</code> - Run once per day<br/>
                                 <code>weekly</code> - Run once per week<br/>
                                 <code>0 */6 * * *</code> - Every 6 hours (cron format)<br/>
-                                <code>0 0 * * 0</code> - Every Sunday at midnight (cron format)
+                                <code>0 0 * * 0</code> - Every Sunday at midnight (cron format)<br/>
+                                <a href="https://crontab.guru/" target="_blank" rel="noopener">Cron Format Helper</a>
                             </div>
                         </div>
 
@@ -1093,6 +1100,75 @@
                 }
 
                 urlTextarea.addEventListener('input', updateURLPreview);
+                const fileInput = document.getElementById('url-file-input');
+                const uploadButton = document.getElementById('url-upload-button');
+                const uploadStatus = document.getElementById('url-upload-status');
+                const dropZone = document.querySelector('.url-editor-column');
+                let pendingFileReads = 0;
+                let fileReadQueue = Promise.resolve();
+
+                function setUploadStatus(message, warning = false) {
+                    uploadStatus.textContent = message;
+                    uploadStatus.classList.toggle('url-upload-warning', warning);
+                }
+
+                function importURLFiles(files) {
+                    if (!files.length) return;
+                    pendingFileReads++;
+                    setUploadStatus('Reading files…');
+                    const errors = [];
+                    fileReadQueue = fileReadQueue.then(async function() {
+                        for (const file of files) {
+                            try {
+                                const text = await file.text();
+                                if (/[\u0000-\u0008\u000e-\u001f]/.test(text)) {
+                                    errors.push(`${file.name}: could not read as text. Use a text or bookmarks export.`);
+                                    continue;
+                                }
+                                if (!extractURLs(text).length) {
+                                    errors.push(`${file.name}: no URLs found.`);
+                                }
+                                if (!text) continue;
+                                urlTextarea.value += (urlTextarea.value && !urlTextarea.value.endsWith('\n') ? '\n' : '') + text;
+                                urlTextarea.dispatchEvent(new Event('input', { bubbles: true }));
+                                urlTextarea.dispatchEvent(new Event('change', { bubbles: true }));
+                            } catch (error) {
+                                errors.push(`${file.name}: could not import this file. Try selecting it again.`);
+                            }
+                        }
+                        urlTextarea.focus();
+                    }).catch(function() {
+                        errors.push('File import failed. Try selecting the files again.');
+                    }).finally(function() {
+                        pendingFileReads--;
+                        setUploadStatus(
+                            errors.length ? errors.join(' ') : (pendingFileReads ? 'Reading files…' : 'File text added.'),
+                            errors.length > 0,
+                        );
+                    });
+                }
+
+                uploadButton.addEventListener('click', () => fileInput.click());
+                fileInput.addEventListener('change', function() {
+                    const files = Array.from(fileInput.files);
+                    fileInput.value = '';
+                    importURLFiles(files);
+                });
+                document.addEventListener('dragover', function(event) {
+                    if (!Array.from(event.dataTransfer.types).includes('Files')) return;
+                    event.preventDefault();
+                    event.dataTransfer.dropEffect = 'copy';
+                    dropZone.classList.add('url-file-dragover');
+                });
+                document.addEventListener('dragleave', function(event) {
+                    if (!event.relatedTarget) dropZone.classList.remove('url-file-dragover');
+                });
+                document.addEventListener('drop', function(event) {
+                    dropZone.classList.remove('url-file-dragover');
+                    if (!Array.from(event.dataTransfer.types).includes('Files')) return;
+                    event.preventDefault();
+                    importURLFiles(Array.from(event.dataTransfer.files));
+                });
                 urlTextarea.addEventListener('scroll', syncHighlightScroll);
                 window.addEventListener('resize', syncHighlightMetrics);
                 window.addEventListener('resize', syncPreviewPanelHeight);
@@ -1306,6 +1382,11 @@
 
                 // Form submission handler
                 document.getElementById('add-form').addEventListener('submit', function(event) {
+                    if (pendingFileReads) {
+                        event.preventDefault();
+                        setUploadStatus('Please wait for the files to finish reading.', true);
+                        return;
+                    }
                     document.getElementById('in-progress').style.display = 'block'
                     document.getElementById('add-form').style.display = 'none'
                     document.getElementById('delay-warning').style.display = 'block'

archivebox/templates/core/setup_wizard.html

@@ -50,7 +50,7 @@
       <div class="abx-question-options" id="archivebox-setup-tls-options">
         <label class="abx-question-option"><input type="radio" name="archivebox-tls-mode" value="localhost"> <span><b>⭐ Localhost</b><small>No setup needed. HTTPS is not required for local access; all replay features remain available.</small></span></label>
         <label class="abx-question-option"><input type="radio" name="archivebox-tls-mode" value="none"> <span><b>⚠️ No TLS, private networks only</b><small>Access ArchiveBox directly on a trusted LAN, VPN, or private network without an ingress service. Public servers require HTTPS. In-browser WARC viewing will be disabled unless using <code>localhost</code> or HTTPS.</small></span></label>
-        <label class="abx-question-option"><input type="radio" name="archivebox-tls-mode" value="single"> <span><b>⚠️ Single-domain HTTPS certificate</b><small>Configure one certificate for the <code>BASE_URL</code> hostname in Cloudflare, Nginx Proxy Manager, Caddy, Traefik, Tailscale, or your platform's ingress UI, then proxy it to ArchiveBox on port <code>8000</code>. This mode is not allowed unless also using Single-domain DNS. Restart ArchiveBox after saving an HTTPS <code>BASE_URL</code> for the first time.</small></span></label>
+        <label class="abx-question-option"><input type="radio" name="archivebox-tls-mode" value="single"> <span><b>⚠️ Single-domain HTTPS certificate</b><small>Configure one certificate for the <code>BASE_URL</code> hostname in Cloudflare, Nginx Proxy Manager, Caddy, Traefik, Tailscale, or your platform's ingress UI, then proxy it to ArchiveBox on port <code>5797</code>. This mode is not allowed unless also using Single-domain DNS. Restart ArchiveBox after saving an HTTPS <code>BASE_URL</code> for the first time.</small></span></label>
         <label class="abx-question-option"><input type="radio" name="archivebox-tls-mode" value="wildcard"> <span><b>⭐ Wildcard TLS</b><small>Configure one certificate covering both the <code>BASE_URL</code> hostname and <code>*.BASE_URL</code>, normally through DNS-01. Never enable on-demand TLS or request individual certificates for snapshot subdomains. Restart ArchiveBox after saving an HTTPS <code>BASE_URL</code> for the first time.</small></span></label>
       </div>
       <div class="abx-question-status" id="archivebox-setup-tls-status">Choose how HTTPS will reach ArchiveBox.</div>

archivebox/templates/core/snapshot.html

@@ -728,14 +728,20 @@
                     grid-column: 1;
                     grid-row: 1;
                     display: grid;
-                    grid-template-columns: auto minmax(0, 1fr);
+                    grid-template-columns: minmax(0, 1fr) auto;
                     align-items: stretch;
                     gap: 4px;
                     font-size: 11px;
                     flex-wrap: nowrap;
                     height: 100%;
                 }
+                .archive-org-label {
+                    display: none;
+                }
                 header .external-links a {
+                    min-width: 0;
+                    white-space: nowrap;
+                    overflow: hidden;
                     justify-content: center;
                     height: 100%;
                     min-height: 32px;
@@ -772,9 +778,6 @@
                 overflow: hidden;
                 min-height: 130px;
             }
-            .header-bottom-frames .card:has([data-compact]) {
-                min-height: 0;
-            }
             .card h4 {
                 font-size: 0.8em;
                 display: inline-block;
@@ -1088,14 +1091,6 @@
             .thumb-card:not([data-plugin-name]) {
                 order: 1;
             }
-            .thumb-card:has([data-compact]) {
-                height: 42px;
-                min-height: 42px;
-                max-height: 42px;
-                grid-row: span 1;
-                order: 2;
-                contain-intrinsic-size: 42px;
-            }
             .thumb-card .thumb-body {
                 display: grid;
                 grid-template-columns: minmax(0, 1fr) auto;
@@ -1158,16 +1153,22 @@
             .thumb-card .thumb-body h4 {
                 display: flex;
                 align-items: center;
-                gap: 2px;
-                font-size: 0.8em;
-                text-transform: uppercase;
-                margin: 0 0 2px 0;
-                color: #222;
-                line-height: 1.1;
+                gap: 3px;
+                font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+                font-size: var(--card-title-size, 11px);
+                font-weight: 600;
+                line-height: 1.2;
+                margin: 0;
+                color: var(--card-title-color, #222);
                 overflow: hidden;
                 text-overflow: ellipsis;
                 white-space: nowrap;
             }
+            .thumb-card .card-title-size {
+                margin-left: auto;
+                flex-shrink: 0;
+                padding-left: 4px;
+            }
 .thumb-card .thumb-body h4 .abx-plugin-icon {
     flex: 0 0 20px;
     align-self: center;
@@ -1219,11 +1220,13 @@
                 transform: none !important;
                 margin: 0 !important;
             }
+            .stack-cover iframe,
             .thumb-card .thumbnail-wrapper iframe,
             .thumb-card .thumbnail-wrapper object,
             .thumb-card .thumbnail-wrapper embed {
                 width: 405% !important;
                 height: 405% !important;
+                max-width: none !important;
                 transform: scale(0.25);
                 transform-origin: 0 0;
                 margin: 0 !important;
@@ -1252,42 +1255,6 @@
             .thumb-card .thumbnail-wrapper > :not(.thumbnail-click-overlay) * {
                 pointer-events: none !important;
             }
-            .thumb-card:has([data-compact]) .thumbnail-wrapper,
-            .thumb-card:has([data-compact]) .thumbnail-wrapper.compact {
-                height: 20px;
-                flex: 0 0 auto;
-            }
-            .thumb-card:has([data-compact]) .thumb-body {
-                min-height: 22px;
-                padding: 2px 4px 2px 6px;
-                font-size: 12px;
-                max-height: 24px;
-            }
-            .thumb-card:has([data-compact]) .thumb-actions a,
-            .thumb-card:has([data-compact]) .thumb-actions button {
-                width: 18px;
-                height: 18px;
-                border-radius: 4px;
-            }
-            .thumb-card:has([data-compact]) .thumb-actions .delete-pending {
-                width: auto;
-            }
-            .thumb-card:has([data-compact]) .thumb-body h4 {
-                font-size: 0.9em;
-                margin-bottom: 0px;
-                line-height: 1;
-                display: flex;
-                align-items: center;
-                gap: 4px;
-                overflow: hidden;
-                text-overflow: ellipsis;
-                white-space: nowrap;
-            }
-            .thumb-card:has([data-compact]) .thumbnail-text-header,
-            .thumb-card:has([data-compact]) .thumbnail-compact-icon,
-            .thumb-card:has([data-compact]) .thumbnail-compact-label {
-                display: none;
-            }
             .thumb-card.selected-card {
                 border: 2px solid orange;
                 box-shadow: 0px -6px 13px 1px rgba(0,0,0,0.05);
@@ -1325,6 +1292,7 @@
             .failed-items a:hover {
                 background: rgba(185, 28, 28, 0.16);
             }
+            .header-bottom[hidden] { display: none; }
             .header-bottom {
                 border-top: 1px solid rgba(170, 30, 85, 0.9);
                 padding-bottom: 1px;
@@ -1412,18 +1380,6 @@
                 margin-bottom: 6px;
                 vertical-align: top;
             }
-            .header-bottom-frames .card:has([data-compact]) .thumbnail-wrapper,
-            .header-bottom-frames .card:has([data-compact]) .thumbnail-wrapper.compact {
-                height: 32px;
-            }
-            .header-bottom-frames .card:has([data-compact]) .thumbnail-text {
-                height: auto;
-                max-height: 64px;
-            }
-            .header-bottom-frames .card:has([data-compact]) .card-body {
-                padding: 4px 8px;
-                max-height: 44px;
-            }
             .header-bottom-frames .card-title {
                 width: 100%;
                 text-align: center;
@@ -1472,6 +1428,7 @@
                 }
             }
         </style>
+        {% include "core/snapshot_stacks.css.html" %}
     </head>
     <body>
         <div id="main-frame-wrapper" class="full-page-wrapper" data-has-outputs="{{ has_outputs|yesno:'1,0' }}" data-snapshot-state="{{ snapshot_state }}">
@@ -1484,10 +1441,10 @@
             {% endif %}
             {% if has_outputs %}
             <iframe id="main-frame"
-                    sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms"
+                    sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms allow-downloads allow-popups allow-popups-to-escape-sandbox"
                     class="full-page-iframe"
                     src="about:blank"
-                    data-default-src="{% if best_result.path %}{% snapshot_preview_url snapshot best_result.path %}{% elif STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}"
+                    data-default-src="{% if best_result.direct_preview_path %}{% snapshot_url snapshot best_result.direct_preview_path %}{% elif best_result.path %}{% snapshot_preview_url snapshot best_result.path %}{% elif STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}"
                     name="preview"
                     loading="eager"
                     fetchpriority="high"></iframe>
@@ -1711,93 +1668,18 @@
                         <div class="external-links">
                             <a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}" title="Browse the full SNAP_DIR for this snapshot" target="_blank">📁 {% if STATIC_EXPORT %}Snapshot manifest{% else %}See all files...{% endif %}</a>
                             <span class="external-links-separator">|</span>
-                            <a href="https://web.archive.org/web/{{url}}" title="Search for a copy of the URL saved in Archive.org" target="_blank" rel="noreferrer">🏛️ Archive.org</a>
+                            <a href="https://web.archive.org/web/{{url}}" aria-label="Search Archive.org" title="Search for a copy of the URL saved in Archive.org" target="_blank" rel="noreferrer"><span aria-hidden="true">🏛️</span><span class="archive-org-label">&nbsp;Archive.org</span></a>
                             <!--<a href="https://archive.md/{{url}}" title="Search for a copy of the URL saved in Archive.today" target="_blank" rel="noreferrer">Archive.today</a>  &nbsp;|&nbsp; -->
                             <!--<a href="https://ghostarchive.org/search?term={{url}}" title="Search for a copy of the URL saved in GhostArchive.org" target="_blank" rel="noreferrer">More...</a>-->
                         </div>
                     </div>
                 </div>
             </div>
-            <div class="header-bottom container-fluid">
+            <div class="header-bottom container-fluid" hidden>
                 <div class="thumb-grid">
                     
                     
-                    {% for result in archiveresults %}
-                        {% with display_path=result.path display_url='' preview_url='' %}
-                        {% if display_path %}{% snapshot_url snapshot display_path as display_url %}{% endif %}
-                        {% if display_path %}{% snapshot_preview_url snapshot display_path result.result as preview_url %}{% endif %}
-                        <div class="thumb-card{% if forloop.first %} selected-card{% endif %}" data-plugin-name="{{result.name|plugin_name}}"{% if preview_url %} data-preview-url="{{preview_url}}"{% endif %}{% if display_path %} data-output-path="{{display_path}}"{% endif %}>
-                                <div class="thumb-body">
-                                    <div class="thumb-actions">
-                                        {% if not STATIC_EXPORT %}<a href="{% snapshot_url snapshot result.name %}/?files=1" data-no-preview="1" title="Open output folder" target="_blank" rel="noopener">📁</a>{% endif %}
-                                        {% if display_path %}
-                                            <a href="{{display_url}}" data-no-preview="1" title="Download output file" download>⬇️</a>
-                                        {% endif %}
-                                        {% if can_delete_outputs and result.result %}
-                                            <button type="button" data-no-preview="1" data-archive-result-ids="{{result.result_ids}}" data-delete-url="{% admin_base_url %}/admin/core/archiveresult/" data-delete-handoff="1" data-delete-snapshot-id="{{snapshot.id}}" title="Delete this output">×</button>
-                                        {% endif %}
-                                    </div>
-                                    {% if display_path %}
-                                        <a href="{{preview_url}}" target="preview" title="./{{display_path}} (downloaded {{result.ts}})">
-                                            <h4>{% plugin_icon result.name %} {{result.name|plugin_name|truncatechars:20}}{% if result.size %} <small>({{result.size|filesizeformat}})</small>{% endif %}</h4>
-                                        </a>
-                                    {% else %}
-                                        <h4>{% plugin_icon result.name %} {{result.name|plugin_name|truncatechars:20}}{% if result.size %} <small>({{result.size|filesizeformat}})</small>{% endif %}</h4>
-                                    {% endif %}
-                                    {% if result.result %}
-                                        {% with plugin_base=result.name|plugin_name %}
-                                            {% if plugin_base == 'ytdlp' or plugin_base == 'yt-dlp' or plugin_base == 'youtube-dl' %}
-                                                {% plugin_card result.result %}
-                                            {% endif %}
-                                        {% endwith %}
-                                    {% endif %}
-                                </div>
-                                {% if result.result and display_path %}
-                                    {% with plugin_base=result.name|plugin_name %}
-                                        {% if plugin_base != 'ytdlp' and plugin_base != 'yt-dlp' and plugin_base != 'youtube-dl' %}
-                                            {# Use plugin-specific card template when ArchiveResult is available #}
-                                            <div class="card-img-top thumbnail-wrapper">
-                                                <a class="thumbnail-click-overlay" href="{{preview_url}}" target="preview" title="./{{display_path}} (downloaded {{result.ts}})" aria-label="Preview {{result.name|plugin_name}}"></a>
-                                                {% plugin_card result.result %}
-                                            </div>
-                                        {% else %}
-                                            {# YT-DLP renders its file list in the body #}
-                                        {% endif %}
-                                    {% endwith %}
-                                {% elif result.is_metadata and display_path %}
-                                    <div class="card-img-top thumbnail-wrapper">
-                                        <a class="thumbnail-click-overlay" href="{{preview_url}}" target="preview" title="./{{display_path}}" aria-label="Preview {{result.name|plugin_name}}"></a>
-                                        {% output_card snapshot display_path result.name %}
-                                    </div>
-                                {% elif display_path %}
-                                    {# Fall back to generic iframe for filesystem-discovered files #}
-                                    <div class="card-img-top thumbnail-wrapper">
-                                        <a class="thumbnail-click-overlay" href="{{preview_url}}" target="preview" title="./{{display_path}}" aria-label="Preview {{result.name|plugin_name}}"></a>
-                                        <iframe class="card-img-top" src="{{display_url}}?autoplay=0" allow="autoplay 'none'; fullscreen 'none'; navigation-override 'none'; " sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy" fetchpriority="low"></iframe>
-                                    </div>
-                                {% endif %}
-                        </div>
-                        {% endwith %}
-                    {% endfor %}
-                    {% if loose_items %}
-                    <div class="thumb-card">
-                            <div class="thumb-body">
-                                <div class="thumb-actions">
-                                    <a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}" data-no-preview="1" title="{% if STATIC_EXPORT %}Open snapshot manifest{% else %}Browse all snapshot files{% endif %}" target="_blank" rel="noopener">📁</a>
-                                </div>
-                                <h4>📦 Other files</h4>
-                                <div class="loose-items">
-                                    {% for item in loose_items %}
-                                        {% if item.is_dir %}
-                                            <a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_url snapshot item.path %}/?files=1{% endif %}" data-no-preview="1" target="_blank" rel="noopener">📁 {{item.name}}</a>
-                                        {% else %}
-                                            <a href="{% snapshot_url snapshot item.path %}" data-no-preview="1" target="_blank" rel="noopener">📄 {{item.name}}</a>
-                                        {% endif %}
-                                    {% endfor %}
-                                </div>
-                            </div>
-                    </div>
-                    {% endif %}
+                    {% include "core/snapshot_output_cards.html" %}
                     {% if failed_items %}
                     <div class="thumb-card">
                             <div class="thumb-body">
@@ -1821,6 +1703,7 @@
             </div>
         </header>
 
+        {{ output_groups|json_script:"snapshot-output-groups" }}
         {% if can_delete_outputs %}{% include "includes/output_delete_controls.html" %}{% endif %}
 
         <script>
@@ -1857,7 +1740,7 @@
 
             function resetPreviewFrameSize(frame) {
                 const wrapper = document.getElementById('main-frame-wrapper')
-                if (!wrapper || !frame) return
+                if (!wrapper || !frame || frame.id !== 'main-frame') return
                 wrapper.style.height = ''
                 wrapper.style.overflow = ''
                 frame.style.height = ''
@@ -1865,7 +1748,7 @@
 
             function fitPreviewFrameToContent(frame) {
                 const wrapper = document.getElementById('main-frame-wrapper')
-                if (!wrapper || !frame || frame.src.includes('.pdf')) return
+                if (!wrapper || !frame || frame.id !== 'main-frame' || frame.src.includes('.pdf')) return
                 try {
                     const doc = frame.contentDocument || frame.contentWindow.document
                     if (!doc || !doc.documentElement) return
@@ -1892,7 +1775,11 @@
                 resetPreviewFrameSize(frame)
                 if (new URL(frame.src || 'about:blank', window.location.href).pathname.endsWith('.pdf')) {
                     frame.removeAttribute('sandbox')
-                    frame.src = frame.src + '#toolbar=0'
+                    const pdfUrl = new URL(frame.src, window.location.href)
+                    const options = new URLSearchParams(pdfUrl.hash.slice(1))
+                    options.set('toolbar', '0')
+                    pdfUrl.hash = options.toString()
+                    if (frame.src !== pdfUrl.href) frame.src = pdfUrl.href
                 }
                 frame.onload = function() {
                     if (new URL(this.src || 'about:blank', window.location.href).pathname.endsWith('.pdf')) {
@@ -1901,7 +1788,7 @@
                     tryCenterImageFrame(this)
                     try {
                         // doesnt work if frame origin rules prevent accessing its DOM via JS
-                        this.contentWindow.scrollTo(0, 0);
+                        if (!new URL(this.src, window.location.href).hash) this.contentWindow.scrollTo(0, 0);
                     } catch(err) {}
                     requestAnimationFrame(() => {
                         fitPreviewFrameToContent(this)
@@ -1935,7 +1822,9 @@
                 if (!card) {
                     return
                 }
-                setPreviewResourcePriority(card, highPriority)
+                // Thumbnail frames stay native-lazy even when their output is selected.
+                // The independent main viewer is already eager/high priority.
+                setPreviewResourcePriority(card, !!card.closest('.stack-cover'))
             }
 
             function prepareThumbnailPriorities() {
@@ -2021,7 +1910,7 @@
                 frame.id = 'main-frame'
                 frame.name = 'preview'
                 frame.className = 'full-page-iframe'
-                frame.sandbox = "allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms"
+                frame.sandbox = "allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms allow-downloads allow-popups allow-popups-to-escape-sandbox"
                 frame.loading = 'eager'
                 frame.setAttribute('fetchpriority', 'high')
                 attachPreviewFrameHandlers(frame)
@@ -2031,9 +1920,6 @@
             function ensureMainFrame(forceReplace=false) {
                 let frame = document.getElementById('main-frame')
                 const wrapper = document.getElementById('main-frame-wrapper')
-                if (wrapper && wrapper.querySelector('#snapshot-empty-state')) {
-                    return null
-                }
                 if (!frame || forceReplace) {
                     const previousFrame = frame
                     frame = createMainFrame(previousFrame)
@@ -2056,20 +1942,20 @@
                 return frame
             }
 
-            function activateCardPreview(card, link, updateHash=true) {
-                if (!card) {
+            function activateCardPreview(card, link, updateHash=true, explicitTarget='') {
+                if (!card && !explicitTarget) {
                     return false
                 }
                 prioritizeCardThumbnail(card, true)
-                const previewUrl = card.dataset.previewUrl
-                const rawTarget = previewUrl || (link ? link.getAttribute('href') : '') || ''
+                const previewUrl = card?.dataset.previewUrl
+                const rawTarget = explicitTarget || (link ? link.getAttribute('href') : '') || previewUrl || ''
                 const target = resolvePreviewUrl(rawTarget)
                 if (!target || target.endsWith('#')) {
                     return false
                 }
 
                 document.querySelectorAll('.selected-card').forEach((selected) => selected.classList.remove('selected-card'))
-                card.closest('.thumb-card').classList.add('selected-card')
+                card?.closest('.thumb-card').classList.add('selected-card')
 
                 const nextSrc = isSnapshotRootPreview(target) ? snapshotFilesUrl : target
                 const existingFrame = document.getElementById('main-frame')
@@ -2088,7 +1974,7 @@
                 if (new URL(target, window.location.href).pathname.endsWith('.pdf')) {
                     iframe_elem.removeAttribute('sandbox')
                 } else {
-                    iframe_elem.sandbox = "allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms"
+                    iframe_elem.sandbox = "allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms allow-downloads allow-popups allow-popups-to-escape-sandbox"
                 }
                 if (rawTarget && updateHash) {
                     window.location.hash = getPreviewHashValueFromHref(rawTarget)
@@ -2124,21 +2010,32 @@
                     return null
                 }
                 const previewLinks = [...document.querySelectorAll('a[target=preview]')]
-                const pathMatch = previewLinks.find((link) => getPreviewHashValue(link) == hashValue)
+                const pathMatch = previewLinks.find((link) => getPreviewHashValueFromHref(link.getAttribute('href')) === hashValue)
                 if (pathMatch) {
                     return pathMatch
                 }
                 const pluginCard = [...document.querySelectorAll('.thumb-card[data-plugin-name]')].find(
-                    (card) => (card.dataset.pluginName || '').toLowerCase() == hashValue,
+                    (card) => (card.dataset.pluginName || '').toLowerCase() === hashValue.toLowerCase(),
                 )
                 return pluginCard && pluginCard.querySelector('a[target=preview]')
             }
 
-            function selectInitialPreview() {
-                const selectedPreviewHash = window.location.hash ? decodeURIComponent(window.location.hash.slice(1)).toLowerCase() : ''
+            function selectInitialPreview(revealStack=true) {
+                let selectedPreviewHash = window.location.hash.slice(1)
+                try { selectedPreviewHash = decodeURIComponent(selectedPreviewHash) } catch (err) {}
                 const matchingLink = findPreviewLinkForHash(selectedPreviewHash)
-                if (matchingLink) {
-                    return activateCardPreview(matchingLink.closest('.thumb-card'), matchingLink, false)
+                if (selectedPreviewHash) {
+                    // An explicit file hash must never fall back to the default card.
+                    // Preserve path case, query parameters and the nested document fragment.
+                    const path = selectedPreviewHash.split(/[?#]/)[0].replace(/^\/+/, '')
+                    const card = matchingLink?.closest('.thumb-card') || [...document.querySelectorAll('.thumb-card[data-output-path]')].find(card =>
+                        path === card.dataset.outputPath || path.startsWith(`${card.dataset.outputPath.split('/')[0]}/`))
+                    const target = matchingLink ? matchingLink.getAttribute('href') :
+                        selectedPreviewHash === 'all' ? snapshotFilesUrl : resolvePreviewUrl(selectedPreviewHash)
+                    activateCardPreview(card, matchingLink, false, target)
+                    if (revealStack) showSnapshotHeader()
+                    document.dispatchEvent(new CustomEvent('snapshot-preview-selected', {detail: {card, discover: !card, revealStack: Boolean(revealStack)}}))
+                    return false
                 }
 
                 const selectedCard = document.querySelector('.thumb-card.selected-card') || document.querySelector('.thumb-card')
@@ -2155,19 +2052,10 @@
                 return false
             }
 
-            for (const card of [...document.querySelectorAll('.thumb-card')]) {
-                card.addEventListener('click', function(event) {
-                    if (event.target && event.target.closest('a')) {
-                        return
-                    }
-                    handleCardClick(card, event)
-                })
-                for (const link of card.querySelectorAll('a')) {
-                    link.addEventListener('click', function(event) {
-                        handleCardClick(card, event)
-                    })
-                }
-            }
+            document.addEventListener('click', function(event) {
+                const card = event.target.closest('.thumb-card')
+                if (card?.dataset.outputPath) handleCardClick(card, event)
+            })
 
             function hideSnapshotHeader() {
                 console.log('Collapsing Snapshot header...')
@@ -2191,13 +2079,15 @@
             }
             function loadSnapshotHeaderState() {
                 // collapse snapshot header if user has previously hidden it
-                let snapshotHeaderIsExpanded = 'false'
+                let snapshotHeaderIsExpanded = 'true'
                 try {
-                    snapshotHeaderIsExpanded = localStorage.getItem("archivebox-snapshot-header-visible") || 'false'
+                    snapshotHeaderIsExpanded = localStorage.getItem("archivebox-snapshot-header-visible") || 'true'
                 } catch (e) {
                     console.log('Could not use localStorage to get header collapse state', e)
                 }
-                if (snapshotHeaderIsExpanded === 'false') {
+                if (snapshotHeaderIsExpanded === 'true') {
+                    showSnapshotHeader()
+                } else {
                     hideSnapshotHeader()
                 }
             }
@@ -2214,10 +2104,14 @@
             // Hide or show the header once when its title row or collapse icon is clicked.
             document.querySelectorAll('.header-toggle-trigger').forEach((trigger) => trigger.addEventListener('click', handleSnapshotHeaderToggle))
 
+            {% include "core/snapshot_stacks.js.html" %}
+            initializeOutputStacks()
+            window.addEventListener('hashchange', selectInitialPreview)
+
             // check URL for hash e.g. #git and load relevant preview
+            loadSnapshotHeaderState()
             selectInitialPreview()
             prepareThumbnailPriorities()
-            loadSnapshotHeaderState()
 
             
 

archivebox/templates/core/snapshot_output_cards.html

@@ -0,0 +1,72 @@
+{% load core_tags %}
+                    {% for result in archiveresults %}
+                        {% with display_path=result.path display_url='' preview_url='' %}
+                        {% if display_path %}{% snapshot_url snapshot display_path as display_url %}{% endif %}
+                        {% if display_path %}{% if result.direct_preview_path %}{% snapshot_url snapshot result.direct_preview_path as preview_url %}{% else %}{% snapshot_preview_url snapshot display_path result.result as preview_url %}{% endif %}{% endif %}
+                        <div class="thumb-card{% if forloop.first %} selected-card{% endif %}" data-plugin-name="{{result.name|plugin_name}}" data-output-group="{{result.output_group}}"{% if preview_url %} data-preview-url="{{preview_url}}"{% endif %}{% if display_path %} data-output-path="{{display_path}}"{% endif %}>
+                                <div class="thumb-body">
+                                    <div class="thumb-actions">
+                                        {% if not STATIC_EXPORT %}<a href="{% snapshot_url snapshot result.folder_path|default:result.name %}/?files=1" data-no-preview="1" title="Open output folder" target="_blank" rel="noopener">📁</a>{% endif %}
+                                        {% if display_path %}
+                                            <a href="{{display_url}}" data-no-preview="1" title="Download output file" download>⬇️</a>
+                                        {% endif %}
+                                        {% if can_delete_outputs and result.result %}
+                                            <button type="button" data-no-preview="1" data-archive-result-ids="{{result.result_ids}}" data-delete-url="{% admin_base_url %}/admin/core/archiveresult/" data-delete-handoff="1" data-delete-snapshot-id="{{snapshot.id}}" title="Delete this output">×</button>
+                                        {% endif %}
+                                    </div>
+                                    {% if display_path %}
+                                        <a href="{{preview_url}}" target="preview" title="./{{display_path}} (downloaded {{result.ts}})">
+                                            <h4><span class="card-title-icon">{% plugin_icon result.name %}</span> {{result.name|plugin_name|truncatechars:20}}{% if result.size %} <small class="card-title-size">{{result.size|file_size:0}}</small>{% endif %}</h4>
+                                        </a>
+                                    {% else %}
+                                        <h4><span class="card-title-icon">{% plugin_icon result.name %}</span> {{result.name|plugin_name|truncatechars:20}}{% if result.size %} <small class="card-title-size">{{result.size|file_size:0}}</small>{% endif %}</h4>
+                                    {% endif %}
+                                </div>
+                                {% if result.result and display_path %}
+                                    <div class="card-img-top thumbnail-wrapper">
+                                        {% with plugin_base=result.name|plugin_name %}
+                                        {% if plugin_base != 'ytdlp' and plugin_base != 'yt-dlp' and plugin_base != 'youtube-dl' %}
+                                        <a class="thumbnail-click-overlay" href="{{preview_url}}" target="preview" title="./{{display_path}} (downloaded {{result.ts}})" aria-label="Preview {{result.name|plugin_name}}"></a>
+                                        {% endif %}
+                                        {% endwith %}
+                                        {% plugin_card result.result %}
+                                    </div>
+                                {% elif result.is_metadata and display_path %}
+                                    <div class="card-img-top thumbnail-wrapper">
+                                        <a class="thumbnail-click-overlay" href="{{preview_url}}" target="preview" title="./{{display_path}}" aria-label="Preview {{result.name|plugin_name}}"></a>
+                                        {% output_card snapshot display_path result.name %}
+                                    </div>
+                                {% elif display_path %}
+                                    {# Fall back to generic iframe for filesystem-discovered files #}
+                                    <div class="card-img-top thumbnail-wrapper">
+                                        <a class="thumbnail-click-overlay" href="{{preview_url}}" target="preview" title="./{{display_path}}" aria-label="Preview {{result.name|plugin_name}}"></a>
+                                        <iframe class="card-img-top" src="{{display_url}}?autoplay=0" allow="autoplay 'none'; fullscreen 'none'; navigation-override 'none'; " sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy" fetchpriority="low"></iframe>
+                                    </div>
+                                {% endif %}
+                        </div>
+                        {% endwith %}
+                    {% endfor %}
+                    {% if loose_items or not STATIC_EXPORT %}
+                    <div class="thumb-card" data-other-files{% if not STATIC_EXPORT %} data-discover-files{% endif %}>
+                            <div class="thumb-body">
+                                <div class="thumb-actions">
+                                    <a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}" data-no-preview="1" title="{% if STATIC_EXPORT %}Open snapshot manifest{% else %}Browse all snapshot files{% endif %}" target="_blank" rel="noopener">📁</a>
+                                </div>
+                                <h4><span class="card-title-icon">📦</span> Other files</h4>
+                            </div>
+                            <div class="thumbnail-wrapper other-files-preview">
+                                <small class="discovery-status" aria-live="polite"></small>
+                                <div class="loose-items">
+                                    {% for item in loose_items %}
+                                        {% if item.is_dir %}
+                                            <a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_url snapshot item.path %}/?files=1{% endif %}" data-no-preview="1" target="_blank" rel="noopener">📁 {{item.name}}</a>
+                                        {% else %}
+                                            <a href="{% snapshot_url snapshot item.path %}" data-no-preview="1" target="_blank" rel="noopener">📄 {{item.name}}</a>
+                                        {% endif %}
+                                    {% empty %}
+                                        <a href="{% snapshot_base_url snapshot %}/?files=1" data-no-preview="1" target="_blank" rel="noopener">📁 Browse all files</a>
+                                    {% endfor %}
+                                </div>
+                            </div>
+                    </div>
+                    {% endif %}

archivebox/templates/core/snapshot_stacks.css.html

@@ -0,0 +1,288 @@
+<style>
+/* Inline so portable exports retain the output browser. */
+.snapshot-stacks .header-bottom {
+    background: #f5f3f4;
+    padding: 6px 8px 7px;
+    max-height: 64vh;
+    overflow: auto;
+    border-bottom: 1px solid #ded5da;
+}
+.snapshot-stacks [hidden] {
+    display: none!important;
+}
+.stack-browser {
+    color: #352b30;
+    font-family: -apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;
+}
+.stack-shelf {
+    display: grid;
+    grid-template-columns: repeat(var(--stack-columns),minmax(0,1fr));
+    gap: 12px;
+    padding: 0 2px;
+}
+.output-stack {
+    border: 0;
+    background: transparent;
+    padding: 0;
+    text-align: left;
+    min-width: 0;
+    color: inherit;
+    font: inherit;
+    cursor: pointer;
+}
+.stack-heading {
+    display: flex;
+    align-items: center;
+    justify-content: flex-start;
+    gap: 5px;
+    margin-bottom: 10px;
+    font-size: 11px;
+}
+.stack-heading strong {
+    font-weight: 650;
+}
+.stack-count {
+    display: inline-flex;
+    align-items: center;
+    justify-content: center;
+    min-width: 16px;
+    height: 16px;
+    padding: 0 4px;
+    background: #e8e1e4;
+    border-radius: 10px;
+    color: #7d6671;
+    font-size: 9px;
+}
+.stack-pile {
+    position: relative;
+    display: block;
+    height: 158px;
+    margin: 0 6px 0 0;
+    perspective: 500px;
+}
+.stack-sheet {
+    display: flex;
+    flex-direction: column;
+    position: absolute;
+    inset: 0;
+    border: 1px solid #d9cdd3;
+    border-radius: 7px;
+    box-shadow: 0 2px 5px #35202d12;
+    background: #fff;
+    overflow: hidden;
+    transition: transform .2s ease,box-shadow .2s ease;
+    transform-origin: bottom center;
+}
+.stack-sheet-2 {
+    transform: translate(5px,-5px) rotate(1.6deg);
+    background: #ece5e8;
+}
+.stack-sheet-1 {
+    transform: translate(2px,-3px) rotate(.8deg);
+    background: #f7f2f4;
+}
+.stack-sheet-0 {
+    z-index: 2;
+}
+.output-stack:hover .stack-sheet-0 {
+    transform: translateY(-3px);
+    box-shadow: 0 5px 12px #35202d26;
+}
+.output-stack:hover .stack-sheet-1 {
+    transform: translate(3px,-4px) rotate(1.2deg);
+}
+.output-stack[aria-expanded=true] .stack-sheet-0 {
+    border-color: #ae2459;
+    box-shadow: 0 0 0 2px #ae245922;
+}
+.output-stack[aria-expanded=true] .stack-heading {
+    color: #a51d52;
+}
+.output-stack[aria-expanded=true] .stack-chevron {
+    transform: rotate(180deg);
+}
+.output-stack.has-selection .stack-count {
+    background: #ac2258;
+    color: #fff;
+}
+.stack-sheet-label {
+    display: block;
+    background: #faf8f9;
+    padding: 4px 6px;
+    border-bottom: 1px solid #eee7eb;
+    font-size: 10px;
+    font-weight: 600;
+    color: #6c5862;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+}
+.stack-cover {
+    display: block;
+    height: 100%;
+    width: 100%;
+    overflow: hidden;
+    pointer-events: none;
+    background: #fff;
+}
+.stack-cover .thumb-card {
+    height: 100%;
+    min-height: 0;
+    max-height: 100%;
+    content-visibility: visible;
+    border: 0;
+}
+/* Hide only header chrome on stack covers, including icons with inline display styles. */
+.stack-cover .thumb-body > .thumb-actions,
+.stack-cover .thumb-body h4 small,
+.stack-cover .thumb-body h4 .card-title-icon,
+.stack-cover .thumb-body h4 .abx-plugin-icon {
+    display: none !important;
+}
+.stack-chevron {
+    display: block;
+    font-size: 15px;
+    color: #9c486a;
+    line-height: 12px;
+    transition: transform .2s;
+    margin-left: auto;
+}
+.stack-no-output {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    height: 100%;
+    border: 1px dashed #d1c3ca;
+    color: #a28d97;
+    border-radius: 7px;
+    font-size: 11px;
+}
+.stack-file-symbol {
+    display: block;
+    margin: auto;
+    color: #ad8b9a;
+    font-size: 30px;
+}
+.stack-tray {
+    margin: 16px 0 0;
+    background: #eae3e6;
+    border: 1px solid #ded0d7;
+    border-radius: 9px;
+    padding: 6px;
+    animation: stack-reveal .18s ease;
+    margin-top: 8px;
+    position: relative;
+}
+.stack-tray-cards {
+    display: grid;
+    grid-template-columns: repeat(auto-fill,minmax(200px,1fr));
+    gap: 6px;
+}
+.stack-tray .thumb-card {
+    min-height: 158px;
+    max-height: 158px;
+    height: 158px;
+    border: 1px solid #d5c9ce;
+    background: #fff;
+    border-radius: 7px;
+    content-visibility: visible;
+    grid-row: auto;
+    order: 0;
+}
+.stack-tray .thumb-card.selected-card {
+    border: 2px solid #ac2258;
+    box-shadow: 0 0 0 2px #ac225813;
+}
+.stack-browser .thumb-card .thumb-body {
+    padding: 3px 5px;
+    background: #fff;
+}
+.stack-browser .stack-cover .thumb-body {
+    min-height: 20px;
+    padding: 3px 6px;
+    background: #faf8f9;
+    border-bottom: 1px solid #eee7eb;
+}
+.stack-cover {
+    --card-title-size: 10px;
+    --card-title-color: #6c5862;
+}
+.stack-browser .thumb-card .thumb-body small {
+    display: inline;
+    font-size: 9px;
+}
+.stack-browser .thumb-card .thumbnail-wrapper {
+    display: block;
+    height: 130px!important;
+    min-height: 130px;
+    max-height: 130px!important;
+    overflow: hidden;
+}
+.stack-empty {
+    grid-column: 1/-1;
+    padding: 18px;
+    text-align: center;
+    font-size: 13px;
+    color: #947482;
+}
+.stack-browser button:focus-visible {
+    outline: 3px solid #b64574;
+    outline-offset: 5px;
+}
+.stack-close {
+    border: 0;
+    background: transparent;
+    padding: 0 5px;
+    font-size: 18px;
+    line-height: 20px;
+    color: #795567;
+    cursor: pointer;
+}
+.stack-tray .thumb-actions {
+    gap: 2px;
+}
+.stack-tray .thumb-actions a,.stack-tray .thumb-actions button {
+    width: 18px;
+    height: 18px;
+    font-size: 11px;
+}
+.stack-tray > .stack-close {
+    position: absolute;
+    top: -7px;
+    right: 4px;
+    z-index: 3;
+    width: 18px;
+    height: 14px;
+    padding: 0;
+    border-radius: 4px;
+    background: #eae3e6;
+    font-size: 15px;
+    line-height: 14px;
+}
+@media (max-width:1000px) {
+    .stack-shelf { grid-template-columns:repeat(4,minmax(0,1fr)); }
+}
+@media (max-width:600px) {
+    .stack-shelf { grid-template-columns:repeat(3,minmax(0,1fr)); gap:12px 8px; }
+    .stack-pile { height:88px; }
+    .stack-tray-cards { grid-template-columns:repeat(2,minmax(0,1fr)); }
+    .snapshot-stacks .header-bottom { padding:5px 6px 6px; }
+}
+@keyframes stack-reveal {
+    from { opacity:0; transform:translateY(-5px); }
+    to { opacity:1; transform:translateY(0); }
+}
+@media (prefers-reduced-motion:reduce) {
+    .stack-browser * { animation:none!important; transition:none!important; }
+}
+
+.other-files-preview { padding: 6px; }
+.other-files-preview .discovery-status:empty { display: none; }
+.other-files-preview .loose-items {
+    height: auto;
+    max-height: 116px;
+    align-items: flex-start;
+    align-content: flex-start;
+}
+
+</style>

archivebox/templates/core/snapshot_stacks.js.html

@@ -0,0 +1,198 @@
+function initializeOutputStacks() {
+    const grid = document.querySelector('.thumb-grid');
+    const data = document.getElementById('snapshot-output-groups');
+    if (!grid || !data) return;
+    const cards = [...grid.children];
+    const groups = JSON.parse(data.textContent).map(group => ({...group, cards: []}));
+    for (const card of cards) {
+        (groups.find(group => group.id === card.dataset.outputGroup) || groups.find(group => group.id === 'other')).cards.push(card);
+    }
+    const name = card => card.dataset.pluginName || 'Files';
+    const el = (tag, className, text) => {
+        const node = document.createElement(tag);
+        node.className = className;
+        if (text !== undefined) node.textContent = text;
+        return node;
+    };
+    const panel = el('section', 'stack-browser');
+    panel.setAttribute('aria-label', 'Snapshot output groups');
+    const shelf = el('div', 'stack-shelf');
+    shelf.style.setProperty('--stack-columns', groups.filter(group => group.id !== 'other' || group.cards.length).length);
+    const tray = el('section', 'stack-tray');
+    tray.id = 'stack-tray';
+    tray.hidden = true;
+    const close = el('button', 'stack-close', '⌃');
+    close.title = 'Collapse stack';
+    close.setAttribute('aria-label', 'Collapse stack');
+    close.type = 'button';
+    const trayCards = el('div', 'stack-tray-cards');
+    tray.append(close, trayCards);
+    const parked = el('div', 'stack-parked');
+    parked.hidden = true;
+    function movePreviewNode(node, parent, before = null) {
+        if (parent.moveBefore && node.isConnected && parent.isConnected) parent.moveBefore(node, before);
+        else parent.insertBefore(node, before);
+    }
+    let active = null;
+    const buttons = new Map();
+    function collapse() {
+        if (!active) return;
+        active.cards.forEach(card => movePreviewNode(card, parked));
+        const button = buttons.get(active.id);
+        button.setAttribute('aria-expanded', 'false');
+        tray.hidden = true;
+        active = null;
+    }
+    let discoveryState = 'idle';
+    let pendingPreviewHash = '';
+    async function discoverOtherFiles(group) {
+        const other = group.cards.find(card => card.hasAttribute('data-discover-files'));
+        if (!other || discoveryState === 'loading' || discoveryState === 'done') return;
+        const status = other.querySelector('.discovery-status');
+        discoveryState = 'loading';
+        status.textContent = 'Scanning for orphan files…';
+        status.removeAttribute('title');
+        try {
+            const url = new URL(window.location.href);
+            url.hash = '';
+            url.search = '?discover_outputs=1';
+            const response = await fetch(url, {cache: 'no-store'});
+            if (!response.ok) throw new Error(`HTTP ${response.status}`);
+            const fragment = new DOMParser().parseFromString(await response.text(), 'text/html');
+            if (!fragment.querySelector('[data-other-files]')) throw new Error('Invalid discovery response');
+            const knownPaths = new Set(cards.map(card => card.dataset.outputPath).filter(Boolean));
+            for (const card of fragment.querySelectorAll('.thumb-card[data-output-path]')) {
+                if (knownPaths.has(card.dataset.outputPath)) continue;
+                knownPaths.add(card.dataset.outputPath);
+                card.classList.remove('selected-card');
+                card.dataset.outputGroup = 'other';
+                group.cards.push(card);
+                cards.push(card);
+                setPreviewResourcePriority(card, false);
+                observer.observe(card, {attributes: true, attributeFilter: ['class']});
+                movePreviewNode(card, active === group ? trayCards : parked);
+            }
+            const existingItems = other.querySelector('.loose-items');
+            const knownLinks = new Set([...existingItems.querySelectorAll('a')].map(link => link.href));
+            for (const link of fragment.querySelectorAll('[data-other-files] .loose-items a')) {
+                if (!knownLinks.has(link.href)) {
+                    knownLinks.add(link.href);
+                    existingItems.append(link);
+                }
+            }
+            buttons.get(group.id).querySelector('.stack-count').textContent = String(group.cards.length);
+            buttons.get(group.id).setAttribute('aria-label', `${group.label}, ${group.cards.length} outputs, expand stack`);
+            status.textContent = '';
+            const coverItems = buttons.get(group.id).querySelector('.stack-cover .loose-items');
+            if (coverItems) coverItems.replaceChildren(...[...existingItems.children].map(item => item.cloneNode(true)));
+            discoveryState = 'done';
+            // Resolve the current hash again; navigation may have changed during the scan.
+            if (pendingPreviewHash && pendingPreviewHash === window.location.hash) selectInitialPreview(active === group);
+        } catch (error) {
+            discoveryState = 'error';
+            status.textContent = 'Could not scan files. ';
+            status.title = error.message;
+            const retry = el('button', '', 'Retry');
+            retry.type = 'button';
+            retry.dataset.noPreview = '1';
+            retry.addEventListener('click', () => discoverOtherFiles(group));
+            status.append(retry);
+        }
+    }
+    function expand(group) {
+        if (active === group) { collapse(); return; }
+        collapse();
+        active = group;
+        buttons.get(group.id).setAttribute('aria-expanded', 'true');
+        trayCards.replaceChildren();
+        if (!group.cards.length) {
+            const empty = el('div', 'stack-empty', 'No captures');
+            trayCards.append(empty);
+        }
+        for (const card of group.cards) {
+            movePreviewNode(card, trayCards);
+        }
+        tray.setAttribute('aria-label', group.label);
+        tray.hidden = false;
+        if (group.id === 'other') discoverOtherFiles(group);
+    }
+    for (const group of groups) {
+        if (group.id === 'other' && !group.cards.length) continue;
+        const button = el('button', `output-stack output-stack-${group.id}`);
+        button.type = 'button';
+        button.setAttribute('aria-expanded', 'false');
+        button.setAttribute('aria-controls', tray.id);
+        button.setAttribute('aria-label', `${group.label}, ${group.cards.length} outputs, expand stack`);
+        const heading = el('span', 'stack-heading');
+        heading.append(el('strong', '', group.label), el('span', 'stack-count', String(group.cards.length)));
+        const pile = el('span', 'stack-pile');
+        pile.setAttribute('aria-hidden', 'true');
+        const top = group.cards[0];
+        const availableLayers = Math.min(3, group.cards.length);
+        for (let index = availableLayers - 1; index >= 0; index--) {
+            const layer = el('span', `stack-sheet stack-sheet-${index}`);
+            if (index === 0) {
+                const cover = el('div', 'stack-cover');
+                const card = top.cloneNode(true);
+                card.classList.remove('selected-card');
+                card.inert = true;
+                for (const key of Object.keys(card.dataset)) delete card.dataset[key];
+                card.querySelectorAll('[id]').forEach(node => node.removeAttribute('id'));
+                // Visible stack covers load the same card document as the expanded tray.
+                setPreviewResourcePriority(card, true);
+                cover.append(card);
+                layer.append(cover);
+            } else {
+                layer.append(el('span', 'stack-sheet-label', name(group.cards[index])));
+            }
+            pile.append(layer);
+        }
+        if (!top) pile.append(el('span', 'stack-no-output', 'No captures'));
+        heading.append(el('span', 'stack-chevron', '⌄'));
+        button.append(heading, pile);
+        button.addEventListener('click', () => {
+            const opening = active !== group;
+            expand(group);
+            const first = group.cards[0];
+            if (opening && first) {
+                activateCardPreview(first, first.querySelector('a[target=preview]') || first.querySelector('a'));
+            }
+        });
+        buttons.set(group.id, button);
+        shelf.append(button);
+
+    }
+
+    function syncSelection() {
+        const selected = cards.find(card => card.classList.contains('selected-card'));
+        for (const group of groups) buttons.get(group.id)?.classList.toggle('has-selection', group.cards.includes(selected));
+    }
+    document.addEventListener('snapshot-preview-selected', event => {
+        const {card, discover, revealStack} = event.detail;
+        pendingPreviewHash = discover ? window.location.hash : '';
+        if (!revealStack) return;
+        const group = groups.find(group => group.cards.includes(card)) || (discover && groups.find(group => group.id === 'other'));
+        if (!group || !buttons.has(group.id)) return;
+        if (active !== group) expand(group);
+        else if (discover) discoverOtherFiles(group);
+    });
+    close.addEventListener('click', () => {
+        const button = buttons.get(active?.id);
+        collapse();
+        button?.focus();
+    });
+    panel.addEventListener('keydown', event => {
+        if (event.key === 'Escape' && active) { close.click(); event.preventDefault(); }
+    });
+    // Existing card event handlers continue to drive the real snapshot viewer and URL hash.
+    const observer = new MutationObserver(syncSelection);
+    cards.forEach(card => observer.observe(card, {attributes: true, attributeFilter: ['class']}));
+    panel.append(shelf, tray, parked);
+    grid.before(panel);
+    cards.forEach(card => movePreviewNode(card, parked));
+    grid.hidden = true;
+    document.body.classList.add('snapshot-stacks');
+    syncSelection();
+    const otherGroup = groups.find(group => group.id === 'other');
+    if (otherGroup && buttons.has('other')) discoverOtherFiles(otherGroup);
+}

archivebox/templates/core/snapshot_thumbnail.html

@@ -0,0 +1,13 @@
+<span class="snapshot-thumbnail" role="img" aria-label="Preview of {{ host }}" style="display:inline-flex;position:relative;vertical-align:middle;align-items:center;justify-content:center;overflow:hidden;border-radius:6px;width:{{ width }};height:{{ height }};background:hsl({{ hue }} 35% 92%);color:hsl({{ hue }} 35% 30%);">
+  <span style="display:flex;flex-direction:column;align-items:center;gap:4px;max-width:90%;">
+    <span aria-hidden="true" style="font:bold 28px sans-serif;">{{ initial }}</span>
+    <span style="font:10px sans-serif;max-width:100%;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">{{ host }}</span>
+    <span aria-hidden="true" style="font-size:12px;">▤</span>
+  </span>
+  {% if first %}
+  <img src="{{ first.url }}" alt="" loading="lazy" decoding="async" data-candidates="{{ candidates }}" data-index="0"
+       style="position:absolute;inset:0;margin:auto;display:block;border:0;max-width:100%;max-height:100%;width:{% if first.kind == 'favicon' %}28px{% else %}100%{% endif %};height:{% if first.kind == 'favicon' %}28px{% else %}100%{% endif %};object-fit:{% if first.kind == 'favicon' %}contain{% else %}cover{% endif %};object-position:top;background:#f8fafc;"
+       onload="try{const c=JSON.parse(this.dataset.candidates)[Number(this.dataset.index)];if(c.kind==='response'&&(Math.min(this.naturalWidth,this.naturalHeight)&lt;=2||Math.max(this.naturalWidth,this.naturalHeight)&lt;=64)){this.onerror();return}if(c.kind==='favicon'){this.previousElementSibling.style.visibility='hidden'}}catch(e){this.remove()}"
+       onerror="try{const cs=JSON.parse(this.dataset.candidates);const i=Number(this.dataset.index)+1;this.dataset.index=i;if(i>=cs.length){this.remove();return}const c=cs[i];this.style.width=this.style.height=c.kind==='favicon'?'28px':'100%';this.style.objectFit=c.kind==='favicon'?'contain':'cover';this.style.objectPosition=c.kind==='favicon'?'center':'top';this.src=c.url}catch(e){this.remove()}">
+  {% endif %}
+</span>

archivebox/templates/core/system_warnings_banner.html

@@ -33,17 +33,13 @@
       — the host is close to OOM. Consider stopping crawls or scaling up.
     {% elif mode == "high_load" %}
       15-min loadavg <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ load_15 }}</code>
-      exceeds <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ load_threshold }}</code>
-      ({{ cpu_count }} cores × 3) — recent CPU or I/O contention may slow crawls. This average can stay elevated after load subsides; it does not measure queue growth.
     {% elif mode == "unsafe" %}
       ArchiveBox single-domain mode — archived pages share an origin with this site
       <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
     {% elif mode == "base_url_mismatch" %}
-      Browser URL:
-      <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ browser_url }}</code>
-      · Configured BASE_URL:
-      <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ configured_base_url }}</code>
-      — GET access is allowed here, but state-changing requests are accepted only on BASE_URL.
+      Editing disabled from unrecognized URL. Visit
+      <a href="{{ configured_base_url }}" style="color:#fff;text-decoration:underline;text-transform:none;">{{ configured_base_url }}</a>
+      to make changes.
     {% else %}
       Ask an ArchiveBox superuser to finish server setup and pin the canonical URL.
     {% endif %}

archivebox/templates/static/add.css

@@ -348,6 +348,35 @@ select {
   min-width: 0;
 }
 
+.url-file-dragover {
+  outline: 2px dashed #417690;
+  outline-offset: 6px;
+}
+
+.url-upload-controls {
+  display: flex;
+  align-items: center;
+  flex-wrap: wrap;
+  gap: 6px;
+  margin-top: 8px;
+  font-size: 12px;
+  color: #666;
+}
+
+.url-upload-controls button {
+  padding: 3px 8px;
+  border: 1px solid #ccc;
+  border-radius: 4px;
+  background: #f8f8f8;
+  color: #333;
+  font-size: inherit;
+  cursor: pointer;
+}
+
+#url-upload-status.url-upload-warning {
+  color: #ba2121;
+}
+
 .url-editor-shell {
   position: relative;
 }

archivebox/templates/static/admin.css

@@ -1987,7 +1987,7 @@ tbody .output-link:hover {opacity: 1;}
 }
 
 .model-snapshot.change-list .cards .card .card-info.card-meta {
-    grid-template-columns: 18px minmax(0, 1fr) auto;
+    grid-template-columns: minmax(0, 1fr) auto minmax(0, 1fr);
 }
 
 .model-snapshot.change-list .cards .card .card-info.card-meta > a {
@@ -2070,7 +2070,7 @@ tbody .output-link:hover {opacity: 1;}
     }
 
     .model-snapshot.change-list .cards .card .card-info.card-meta {
-        grid-template-columns: 18px minmax(0, 1fr) auto;
+        grid-template-columns: minmax(0, 1fr) auto minmax(0, 1fr);
     }
 }
 

archivebox/templates/static/setup_wizard.js

@@ -195,9 +195,11 @@
     } else {
       risks.push('⚠️ UI, API, and archive replay share one origin. CSP disables risky archived scripts, but a CSP or content-type bypass could expose the archive index, other snapshots, saved headers, admin pages, API data, and canonical-host mutations.');
     }
-    risks.push(publicIndexInput.checked
-      ? '⚠️ Anonymous visitors can enumerate public snapshot URLs and titles; saved URLs may contain private share tokens or other secrets.'
-      : '✅ Anonymous visitors cannot browse the snapshot index.');
+    if (!publicIndexInput.checked) {
+      risks.push('✅ Anonymous visitors cannot browse the snapshot index.');
+    } else {
+      risks.push('⚠️ Anonymous visitors can browse the public snapshot index; saved URLs may contain private share tokens or other secrets.');
+    }
     risks.push(!controlPlaneEnabled
       ? '✅ URL submission and other state-changing requests are disabled for everyone in this replay-only mode.'
       : (publicAddInput.checked ? '⚠️ Anonymous visitors can submit malicious or private/intranet URLs. A filtering or per-crawl configuration bypass could expose internal content or threaten the server.' : '✅ Only signed-in admins can submit new URLs.'));
@@ -234,7 +236,7 @@
     var exampleBaseUrl = desiredScheme + exampleBaseHost;
     var exampleWildcardUrl = desiredScheme + exampleWildcardHost;
     var exampleAdminUrl = desiredScheme + (dnsMode === 'wildcard' ? 'admin.' + exampleBaseHost : exampleBaseHost) + '/admin/';
-    var localhostAdminUrl = 'http://admin.archivebox.localhost:8000/admin/';
+    var localhostAdminUrl = 'http://admin.archivebox.localhost:5797/admin/';
 
     document.getElementById('archivebox-setup-hosting-status').textContent = hosting === 'localhost'
       ? '❌ Visit ' + localhostAdminUrl + ' from the same machine to continue setup.'
@@ -253,9 +255,9 @@
     document.getElementById('archivebox-setup-tls-status').textContent = tlsMode === 'localhost'
       ? '❌ Visit ' + localhostAdminUrl + ' from this machine to continue setup. No certificate is needed.'
       : (tlsMode === 'wildcard'
-        ? '❌ In your ingress provider, proxy to ArchiveBox on port 8000 and configure one browser-trusted certificate covering ' + exampleBaseHost + ' and ' + exampleWildcardHost + ', normally through DNS-01. Never enable on-demand TLS or request individual snapshot certificates. Visit ' + exampleAdminUrl + ' to continue setup.'
+        ? '❌ In your ingress provider, proxy to ArchiveBox on port 5797 and configure one browser-trusted certificate covering ' + exampleBaseHost + ' and ' + exampleWildcardHost + ', normally through DNS-01. Never enable on-demand TLS or request individual snapshot certificates. Visit ' + exampleAdminUrl + ' to continue setup.'
         : (tlsMode === 'single'
-          ? '❌ In your ingress provider, proxy to ArchiveBox on port 8000 and configure one browser-trusted certificate for ' + exampleBaseHost + '. Visit ' + exampleAdminUrl + ' to continue setup.'
+          ? '❌ In your ingress provider, proxy to ArchiveBox on port 5797 and configure one browser-trusted certificate for ' + exampleBaseHost + '. Visit ' + exampleAdminUrl + ' to continue setup.'
           : (tlsMode === 'none'
             ? '❌ Expose this ArchiveBox server directly over HTTP without a separate ingress or SSL termination service. Visit ' + exampleAdminUrl + ' to continue setup. In-browser WARC viewing will remain disabled unless browsing through localhost or HTTPS.'
             : '❌ Choose an ingress and TLS mode. Your choice will not be changed automatically.')));

archivebox/tests/conftest.py

@@ -1164,9 +1164,9 @@ def run_python_cwd(
 # Server/API Integration Helpers
 # =============================================================================
 
-API_TEST_HOST = "api.archivebox.localhost:8000"
-ADMIN_TEST_HOST = "admin.archivebox.localhost:8000"
-WEB_TEST_HOST = "web.archivebox.localhost:8000"
+API_TEST_HOST = "api.archivebox.localhost:5797"
+ADMIN_TEST_HOST = "admin.archivebox.localhost:5797"
+WEB_TEST_HOST = "web.archivebox.localhost:5797"
 
 
 @pytest.fixture
@@ -1794,8 +1794,8 @@ def real_archive_with_example(tmp_path_factory, request):
         [
             "config",
             "--set",
-            "BIND_ADDR=127.0.0.1:8000",
-            "BASE_URL=http://archivebox.localhost:8000",
+            "BIND_ADDR=127.0.0.1:5797",
+            "BASE_URL=http://archivebox.localhost:5797",
             "PUBLIC_INDEX=True",
             "PUBLIC_ADD_VIEW=True",
         ],

archivebox/tests/fixtures/tlsnotary/hacker-news/metadata.json

@@ -0,0 +1 @@
+{"server_name":"news.ycombinator.com","time":1790032281,"status":200,"bytes":6326,"receipt_bytes":434,"extension_version":"0.1.0.1501","verifier_url":"https://tlsnotary.zervice.io","receipt_id":"8f3ddcf72675db324390b63d63bb3a853ad9024f0489572eab7a5434ad697934","mode":"Mpc"}

archivebox/tests/fixtures/tlsnotary/hacker-news/navigation.json

@@ -0,0 +1,8 @@
+{
+  "waitUntil": "domcontentloaded",
+  "elapsed": 123,
+  "url": "https://news.ycombinator.com/",
+  "finalUrl": "https://news.ycombinator.com/",
+  "status": 200,
+  "timestamp": "2026-09-21T23:11:15.783Z"
+}

archivebox/tests/fixtures/tlsnotary/hacker-news/receipt.json

@@ -0,0 +1 @@
+{"payload":"eyJmb3JtYXQiOiJhYngtdGxzbm90YXJ5LXJlY2VpcHQtdjEiLCJzZXJ2ZXJfbmFtZSI6Im5ld3MueWNvbWJpbmF0b3IuY29tIiwidGltZSI6MTc5MDAzMjI4MSwiYWxnb3JpdGhtIjoiU0hBMjU2IiwiaGFzaCI6IjJhZWU0OTRjNTJhNjQyYjhmZWQ4YjQ4MDEwZmQ1MDYxNGE0NGU3ZDlmZTdhYjM4YWM4MDYwNTk2NTdiYzhlNzIiLCJzdGFydCI6MCwiZW5kIjo2MzI2fQ==","signature":"yVEK8gUZoKQtaTALV/Cv9pAUcqxvZH2l37S1/Ckglhb/dHKQGmNtOx5KANOYVT1wa8LrEwzWqkBobfC5TAkRCg==","blinder":"Hs7PDzNY0+xuPyksShpN/A=="}

archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py

@@ -67,3 +67,68 @@ def test_archiveresult_patch_upload_finalizes_queued_result(client, api_admin_us
     assert result.output_str == "output.html"
     assert snapshot.status == Snapshot.StatusChoices.SEALED
     assert snapshot.retry_at is not None
+
+
+@pytest.mark.parametrize("chunked", [False, True])
+@pytest.mark.parametrize("method", ["POST", "PATCH"])
+def test_live_archiveresult_multipart_upload(tmp_path, chunked, method):
+    import requests
+
+    from archivebox.tests.conftest import (
+        api_auth_headers,
+        cli_env,
+        create_admin_and_token,
+        get_free_port,
+        init_archive,
+        live_api_request,
+        stop_archivebox_process,
+    )
+    from archivebox.tests.test_api_v1_cli_add import start_api_server_without_runner
+
+    init_archive(tmp_path)
+    token = create_admin_and_token(tmp_path)
+    port = get_free_port()
+    server = start_api_server_without_runner(tmp_path, cli_env(port=port, server=True), port)
+    try:
+        snapshot = live_api_request(
+            port,
+            "post",
+            "/api/v1/core/snapshots",
+            api_token=token,
+            json={"url": "https://example.com/multipart", "status": "sealed"},
+        )
+        assert snapshot.status_code == 200, snapshot.text
+        result = live_api_request(
+            port,
+            "post",
+            "/api/v1/core/archiveresults",
+            api_token=token,
+            data={"snapshot_id": snapshot.json()["id"], "plugin": "dom", "status": "started"},
+        )
+        assert result.status_code == 200, result.text
+        payload = b"<html>Real multipart upload</html>"
+        body = encode_multipart(
+            BOUNDARY,
+            {
+                "files": SimpleUploadedFile("output.html", payload, content_type="text/html"),
+                "output_paths": "output.html",
+                "status": "succeeded",
+                "snapshot_id": snapshot.json()["id"],
+                "plugin": "dom",
+            },
+        )
+        path = "/api/v1/core/archiveresults" if method == "POST" else f"/api/v1/core/archiveresult/{result.json()['id']}"
+        response = requests.request(
+            method,
+            f"http://127.0.0.1:{port}{path}",
+            data=iter([body]) if chunked else body,
+            headers={**api_auth_headers(token, port=port), "Content-Type": MULTIPART_CONTENT},
+            timeout=30,
+        )
+        assert response.status_code == 200, response.text
+        assert response.json()["status"] == "succeeded"
+        files = list((tmp_path / "archive").glob("users/*/snapshots/*/*/*/dom/output.html"))
+        assert len(files) == 1
+        assert files[0].read_bytes() == payload
+    finally:
+        stop_archivebox_process(server)

archivebox/tests/test_api_v1_core_archiveresults.py

@@ -15,6 +15,79 @@ from archivebox.tests.conftest import api_client_request
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+@pytest.mark.parametrize("plugin", [".", "..", "/./", "/../", " . ", " ../ "])
+def test_archiveresult_upload_rejects_dot_directory_names(client, api_admin_user, api_headers, plugin):
+    crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/invalid-upload-directory", crawl=crawl)
+    snapshot_dir = snapshot.output_dir
+    filename = "rejected-upload.txt"
+
+    response = client.post(
+        "/api/v1/core/archiveresults",
+        {
+            "snapshot_id": str(snapshot.id),
+            "plugin": plugin,
+            "files": SimpleUploadedFile(filename, b"must not be written", content_type="text/plain"),
+        },
+        **api_headers,
+    )
+
+    assert response.status_code == 400, response.content
+    assert response.json()["detail"] == "Invalid ArchiveResult plugin name"
+    assert not snapshot.archiveresult_set.exists()
+    assert not (snapshot_dir / filename).exists()
+    assert not (snapshot_dir.parent / filename).exists()
+
+
+@pytest.mark.parametrize("plugin", [".", ".."])
+def test_archiveresult_patch_rejects_existing_dot_directory_names(client, api_admin_user, api_headers, plugin):
+    crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/invalid-existing-directory", crawl=crawl)
+    result = ArchiveResult.objects.create(snapshot=snapshot, plugin=plugin)
+    snapshot_dir = snapshot.output_dir
+    filename = "rejected-patch.txt"
+
+    response = client.patch(
+        f"/api/v1/core/archiveresult/{result.id}",
+        encode_multipart(
+            BOUNDARY,
+            {"files": SimpleUploadedFile(filename, b"must not be written", content_type="text/plain")},
+        ),
+        content_type=MULTIPART_CONTENT,
+        **api_headers,
+    )
+
+    assert response.status_code == 400, response.content
+    assert response.json()["detail"] == "Invalid ArchiveResult plugin name"
+    result.refresh_from_db()
+    assert result.output_files == {}
+    assert not (snapshot_dir / filename).exists()
+    assert not (snapshot_dir.parent / filename).exists()
+
+
+@pytest.mark.parametrize("plugin", ["custom.files", "custom-files_v1"])
+def test_archiveresult_upload_accepts_unregistered_directory_names(client, api_admin_user, api_headers, plugin):
+    crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/custom-upload-directory", crawl=crawl)
+    contents = b"uploaded original file"
+
+    response = client.post(
+        "/api/v1/core/archiveresults",
+        {
+            "snapshot_id": str(snapshot.id),
+            "plugin": plugin,
+            "files": SimpleUploadedFile("original.txt", contents, content_type="text/plain"),
+        },
+        **api_headers,
+    )
+
+    assert response.status_code == 200, response.content
+    result = snapshot.archiveresult_set.get()
+    assert result.plugin == plugin
+    assert result.output_files["original.txt"]["size"] == len(contents)
+    assert (snapshot.output_dir / plugin / "original.txt").read_bytes() == contents
+
+
 def test_archiveresult_upload_upserts_by_snapshot_plugin_and_hook(client, api_admin_user, api_headers):
     crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
     snapshot = Snapshot.objects.create(url="https://example.com/unified-result", crawl=crawl)

archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py

@@ -151,7 +151,13 @@ def test_snapshot_pause_resume_api_leaves_archiveresult_facts_unchanged(
             expected_exit_codes=(1,),
         )
         assert failed_result.status == ArchiveResult.StatusChoices.FAILED
-        assert failed_result.output_str == "git fetch failed (exit=128)"
+        assert failed_result.output_str == (
+            "git fetch failed (exit=128): fatal: '/nonexistent/archivebox-test-repository.git' "
+            "does not appear to be a git repository\n"
+            "fatal: Could not read from remote repository.\n\n"
+            "Please make sure you have the correct access rights\n"
+            "and the repository exists."
+        )
         install_result = run_archivebox_cmd(["install"], cwd=tmp_path, timeout=600)
         assert install_result.returncode == 0, install_result.stderr or install_result.stdout
         now = timezone.now()

archivebox/tests/test_api_v1_core_snapshots_rss.py

@@ -11,7 +11,7 @@ pytestmark = pytest.mark.django_db
 
 
 User = get_user_model()
-ADMIN_HOST = "admin.archivebox.localhost:8000"
+ADMIN_HOST = "admin.archivebox.localhost:5797"
 
 
 @pytest.fixture

archivebox/tests/test_api_v1_core_tags_autocomplete.py

@@ -12,7 +12,7 @@ def test_basic_success_case_request(client, tmp_path, api_admin_user, api_token)
     response = client.get(
         "/api/v1/core/tags/autocomplete/",
         {"q": "api-basic", "api_key": api_token.token},
-        HTTP_HOST="api.archivebox.localhost:8000",
+        HTTP_HOST="api.archivebox.localhost:5797",
     )
 
     assert response.status_code == 200, response.content

archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py

@@ -33,7 +33,7 @@ from .conftest import (
 
 pytestmark = pytest.mark.django_db(transaction=True)
 User = get_user_model()
-ADMIN_HOST = "admin.archivebox.localhost:8000"
+ADMIN_HOST = "admin.archivebox.localhost:5797"
 
 
 @pytest.fixture

archivebox/tests/test_api_v1_personas_sync.py

@@ -1,3 +1,5 @@
+import json
+
 import pytest
 
 from archivebox.tests.conftest import api_client_request
@@ -23,3 +25,92 @@ def test_basic_success_case_request(client, api_headers):
 
     assert response.status_code == 200, response.content
     assert response.json()["success"] is True
+
+
+def test_sync_preserves_color_scheme_and_clears_explicitly_removed_cookies(client, api_headers):
+    from archivebox.personas.models import Persona
+
+    payload = {
+        "extension_persona_id": "extension-settings-update",
+        "name": "extension-settings-update",
+        "settings": {"color_scheme": "dark"},
+        "cookies_txt": "# Netscape HTTP Cookie File\nexample.com\tFALSE\t/\tTRUE\t0\tsession\ttest-session\n",
+        "auth_json": {"cookies": [{"domain": "example.com", "name": "session", "value": "test-session", "path": "/"}]},
+    }
+    response = api_client_request(client, "post", "/api/v1/personas/sync", payload=payload, headers=api_headers)
+    assert response.status_code == 200, response.content
+    persona = Persona.objects.get(pk=response.json()["persona"]["id"])
+    assert persona.config["BROWSER_COLOR_SCHEME"] == "dark"
+    assert "test-session" in (persona.path / "cookies.txt").read_text()
+    assert json.loads((persona.path / "auth.json").read_text()) == payload["auth_json"]
+    payload["settings"]["color_scheme"] = "light"
+    payload["cookies_txt"] = ""
+    payload["auth_json"] = {}
+    response = api_client_request(client, "post", "/api/v1/personas/sync", payload=payload, headers=api_headers)
+    assert response.status_code == 200, response.content
+    persona.refresh_from_db()
+    assert persona.config["BROWSER_COLOR_SCHEME"] == "light"
+    assert response.json()["created"] is False
+    assert (persona.path / "cookies.txt").read_text() == ""
+    assert json.loads((persona.path / "auth.json").read_text()) == {}
+    assert response.json()["cookies_file_written"] is True
+    assert response.json()["auth_file_written"] is True
+
+
+def test_sync_omitted_auth_artifacts_preserves_existing_cookies(client, api_headers):
+    from archivebox.personas.models import Persona
+
+    payload = {
+        "extension_persona_id": "extension-settings-only",
+        "name": "extension-settings-only",
+        "cookies_txt": "# Netscape HTTP Cookie File\nexample.com\tFALSE\t/\tTRUE\t0\tsession\ttest-session\n",
+        "auth_json": {"cookies": [{"domain": "example.com", "name": "session", "value": "test-session", "path": "/"}]},
+    }
+    response = api_client_request(client, "post", "/api/v1/personas/sync", payload=payload, headers=api_headers)
+    assert response.status_code == 200, response.content
+    persona = Persona.objects.get(pk=response.json()["persona"]["id"])
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/personas/sync",
+        payload={"extension_persona_id": payload["extension_persona_id"], "name": payload["name"], "settings": {"language": "en-US"}},
+        headers=api_headers,
+    )
+    assert response.status_code == 200, response.content
+    assert (persona.path / "cookies.txt").read_text() == payload["cookies_txt"]
+    assert json.loads((persona.path / "auth.json").read_text()) == payload["auth_json"]
+    assert response.json()["cookies_file_written"] is False
+    assert response.json()["auth_file_written"] is False
+
+
+def test_sync_rejects_invalid_color_scheme(client, api_headers):
+    from archivebox.personas.models import Persona
+
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/personas/sync",
+        payload={"extension_persona_id": "invalid-color-scheme", "name": "invalid-color-scheme", "settings": {"color_scheme": "purple"}},
+        headers=api_headers,
+    )
+    assert response.status_code == 422, response.content
+    assert not Persona.objects.filter(name="invalid-color-scheme").exists()
+
+
+def test_sync_explicit_default_color_scheme_resets_override(client, api_headers):
+    from archivebox.personas.models import Persona
+
+    payload = {"extension_persona_id": "color-reset", "name": "color-reset", "settings": {"color_scheme": "dark"}}
+    response = api_client_request(client, "post", "/api/v1/personas/sync", payload=payload, headers=api_headers)
+    assert response.status_code == 200, response.content
+    persona = Persona.objects.get(pk=response.json()["persona"]["id"])
+    payload["settings"] = {}
+    response = api_client_request(client, "post", "/api/v1/personas/sync", payload=payload, headers=api_headers)
+    assert response.status_code == 200, response.content
+    persona.refresh_from_db()
+    assert persona.config["BROWSER_COLOR_SCHEME"] == "dark"
+    payload["settings"] = {"color_scheme": ""}
+    response = api_client_request(client, "post", "/api/v1/personas/sync", payload=payload, headers=api_headers)
+    assert response.status_code == 200, response.content
+    persona.refresh_from_db()
+    assert persona.config["BROWSER_COLOR_SCHEME"] == ""

archivebox/tests/test_archive_result_service.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 from importlib.resources import files
+import hashlib
 import json
 import os
 import shutil
@@ -43,13 +44,15 @@ def _run_shipped_snapshot_hook(
     lib_dir: Path,
     env: dict | None = None,
     expected_exit_codes: tuple[int, ...] = (0,),
+    cancel_when=None,
 ):
     """Run one shipped hook through the production process/result bus services."""
     import asyncio
 
     from abx_dl.services.process_service import ProcessService as HookProcessService
     from abx_dl.services.archive_result_service import ArchiveResultService as HookArchiveResultService
-    from abx_plugins.plugins.base.utils import get_hydrated_required_binaries
+    from abx_plugins.plugins.base.utils import _collect_required_schema_paths, get_hydrated_required_binaries
+    from abxpkg import prepare_script_exec_plan
     from archivebox.core.models import ArchiveResult
     from archivebox.machine.models import Process
     from archivebox.plugins.discovery import get_plugin_catalog
@@ -72,11 +75,19 @@ def _run_shipped_snapshot_hook(
             binproviders=required_binary["binproviders"],
             overrides=required_binary.get("overrides"),
         )
-    binary_env = resolve_abxpkg_binary_env(lib_dir, deps_from=hook_config)
+    binary_env = resolve_abxpkg_binary_env(lib_dir, deps_from=_collect_required_schema_paths(hook_config))
+    hook_env = {
+        **binary_env,
+        "ABXPKG_LIB_DIR": str(lib_dir),
+        "SNAP_DIR": str(snapshot.output_dir),
+        "PATH": f"{Path(os.sys.executable).parent}{os.pathsep}{os.environ['PATH']}",
+        **(env or {}),
+    }
+    assert prepare_script_exec_plan(hook_path, env=hook_env), f"Could not prepare shipped hook {hook_path}"
     output_dir = Path(snapshot.output_dir) / plugin
     output_dir.mkdir(parents=True, exist_ok=True)
     bus = create_bus(name=f"test_real_{plugin}_{snapshot.id}")
-    HookProcessService(bus, emit_jsonl=False, interactive_tty=False)
+    hook_process_service = HookProcessService(bus, emit_jsonl=False, interactive_tty=False)
     HookArchiveResultService(bus, emit_jsonl=False)
     PersistedProcessService(bus)
     ArchiveResultService(bus)
@@ -95,13 +106,7 @@ def _run_shipped_snapshot_hook(
                     hook_name=projected_hook_name,
                     hook_path=str(hook_path),
                     hook_args=[f"--url={snapshot.url}"],
-                    env={
-                        **binary_env,
-                        "ABXPKG_LIB_DIR": str(lib_dir),
-                        "SNAP_DIR": str(snapshot.output_dir),
-                        "PATH": f"{Path(os.sys.executable).parent}{os.pathsep}{os.environ['PATH']}",
-                        **(env or {}),
-                    },
+                    env=hook_env,
                     output_dir=str(output_dir),
                     timeout=60,
                     is_background=".bg." in hook_name,
@@ -112,12 +117,19 @@ def _run_shipped_snapshot_hook(
                 ),
             )
             await process_event.now()
+            if cancel_when is not None:
+                assert await asyncio.to_thread(cancel_when.wait, 30)
+                tasks = list(hook_process_service._background_completion_tasks)
+                assert len(tasks) == 1
+                tasks[0].cancel()
+                outcomes = await asyncio.gather(*tasks, return_exceptions=True)
+                assert isinstance(outcomes[0], asyncio.CancelledError), outcomes[0]
             if ".bg." in hook_name:
                 completed_event = await bus.find(
                     ProcessCompletedEvent,
                     child_of=process_event,
                     past=True,
-                    future=90,
+                    future=False if cancel_when is not None else 90,
                 )
                 assert completed_event is not None
                 await completed_event.wait(timeout=90)
@@ -131,7 +143,8 @@ def _run_shipped_snapshot_hook(
     assert process is not None
     process.refresh_from_db()
     assert process.exit_code in expected_exit_codes, (process.stdout, process.stderr)
-    result = ArchiveResult.objects.get(snapshot=snapshot, plugin=plugin, hook_name=projected_hook_name)
+    results = ArchiveResult.objects.filter(snapshot=snapshot, plugin=plugin, hook_name=projected_hook_name)
+    result = results.first() if cancel_when is not None else results.get()
     return process, result
 
 
@@ -189,15 +202,19 @@ def test_process_completed_projects_inline_archiveresult(tmp_path, hermetic_lib_
 
     assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
     assert result.process_id == process.id
-    assert result.output_str.endswith(json.loads((snapshot_dir / "hashes" / "hashes.json").read_text())["root_hash"][:12])
+    manifest = snapshot_dir / "hashes" / "hashes.json"
+    checksum = snapshot_dir / "hashes" / "hashes.sha256"
+    assert checksum.read_text() == hashlib.sha256(manifest.read_bytes()).hexdigest() + "\n"
+    assert result.output_str.endswith(json.loads(manifest.read_text())["root_hash"][:12])
     assert result.output_files == {
         "hashes.json": {
             "extension": "json",
             "mimetype": "application/json",
-            "size": (snapshot_dir / "hashes" / "hashes.json").stat().st_size,
+            "size": manifest.stat().st_size,
         },
+        "hashes.sha256": {"extension": "sha256", "mimetype": "", "size": 65},
     }
-    assert result.output_size == (snapshot_dir / "hashes" / "hashes.json").stat().st_size
+    assert result.output_size == manifest.stat().st_size + checksum.stat().st_size
     _cleanup_machine_process_rows()
 
 

archivebox/tests/test_auth_ldap.py

@@ -153,7 +153,7 @@ class TestArchiveBoxWithLDAP:
 
     def test_archivebox_init_without_ldap(self, tmp_path):
         """Test that archivebox init works without LDAP enabled."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["init"],
             cwd=tmp_path,
             timeout=45,
@@ -161,14 +161,13 @@ class TestArchiveBoxWithLDAP:
             default_cli_env=True,
             disable_extractors=True,
         )
-        _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         # Should succeed
-        assert code == 0, f"archivebox init failed: {stderr}"
+        assert result.returncode == 0, f"archivebox init failed: {result.stderr}"
 
     def test_archivebox_version_with_ldap_config(self, tmp_path):
         """Test that archivebox version works with LDAP config set."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["version"],
             cwd=tmp_path,
             timeout=10,
@@ -179,10 +178,9 @@ class TestArchiveBoxWithLDAP:
             default_cli_env=True,
             disable_extractors=True,
         )
-        _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         # Should succeed
-        assert code == 0, f"archivebox version failed: {stderr}"
+        assert result.returncode == 0, f"archivebox version failed: {result.stderr}"
 
 
 class TestLDAPConfigValidationInArchiveBox:
@@ -190,7 +188,7 @@ class TestLDAPConfigValidationInArchiveBox:
 
     def test_archivebox_init_with_incomplete_ldap_config(self, tmp_path):
         """Test that archivebox init fails with helpful error when LDAP config is incomplete."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["init"],
             cwd=tmp_path,
             timeout=45,
@@ -201,10 +199,9 @@ class TestLDAPConfigValidationInArchiveBox:
             default_cli_env=True,
             disable_extractors=True,
         )
-        _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         # Should fail with validation error
-        assert code != 0, "Should fail with incomplete LDAP config"
+        assert result.returncode != 0, "Should fail with incomplete LDAP config"
 
         # Check error message
-        assert "LDAP_* config options must all be set" in stderr, f"Expected validation error message in: {stderr}"
+        assert "LDAP_* config options must all be set" in result.stderr, f"Expected validation error message in: {result.stderr}"

archivebox/tests/test_cli_add.py

@@ -6,6 +6,15 @@ Verify add creates snapshots in DB, crawls, source files, and archive directorie
 
 import os
 import json
+import pty
+import select
+import time
+import termios
+import signal
+import re
+import subprocess
+
+import psutil
 from pathlib import Path
 
 import pytest
@@ -27,6 +36,345 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+@pytest.mark.parametrize(
+    "choice,plugins",
+    [(choice, "chrome") for choice in ["skip", "retry", "abort", "ctrl-c", "noninteractive"]] + [("noninteractive", "archivewebpage")],
+)
+def test_add_interrupts_active_hook(initialized_archive, choice, plugins):
+    env = cli_env(CHROME_DELAY_AFTER_LOAD="60", CHROME_TIMEOUT="120", CHROME_HEADLESS="True")
+    # The timed observation below is for a running navigate hook. Resolve the
+    # selected plugin's real binaries before starting it, so a cold install is
+    # not counted as a hook-start failure.
+    installed = run_archivebox_cmd(["install", plugins], cwd=initialized_archive, env=env, timeout=600)
+    assert installed.returncode == 0, installed.stderr or installed.stdout
+
+    master, slave = pty.openpty()
+    termios.tcsetwinsize(slave, (40, 160))
+    output = bytearray()
+    result = None
+
+    def read_until(predicate, timeout=45):
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            if select.select([master], [], [], 0.1)[0]:
+                output.extend(os.read(master, 65536))
+            if predicate():
+                return
+        raise AssertionError(output.decode(errors="replace"))
+
+    def active_hook():
+        with use_archivebox_db(initialized_archive):
+            return Process.objects.filter(
+                process_type=Process.TypeChoices.HOOK,
+                cmd__0__endswith="on_Snapshot__30_chrome_navigate.js",
+                status="running",
+            ).first()
+
+    try:
+        result = run_archivebox_cmd(
+            ["add", f"--plugins={plugins}", "https://example.com"],
+            cwd=initialized_archive,
+            env=env,
+            stdin=subprocess.DEVNULL if choice == "noninteractive" else slave,
+            stdout=slave,
+            stderr=slave,
+            wait=False,
+            start_new_session=True,
+        )
+        read_until(lambda: active_hook() is not None)
+        hook = active_hook()
+        assert hook is not None
+        result.send_signal(signal.SIGINT)
+        if choice != "noninteractive":
+            read_until(lambda: b"Choice [skip]:" in output)
+            read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
+            assert not psutil.pid_exists(hook.pid)
+            if choice == "retry":
+                os.write(master, b"r")
+                read_until(lambda: (new_hook := active_hook()) is not None and new_hook.pid != hook.pid)
+                prompt_offset = len(output)
+                result.send_signal(signal.SIGINT)
+                read_until(lambda: b"Choice [skip]:" in output[prompt_offset:])
+                read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
+            if choice == "ctrl-c":
+                result.send_signal(signal.SIGINT)
+            else:
+                os.write(master, b"a" if choice == "abort" else b"\r")
+        read_until(lambda: result.poll() is not None)
+        assert result.returncode == (130 if choice in {"abort", "ctrl-c", "noninteractive"} else 0), output.decode(errors="replace")
+        if choice == "noninteractive":
+            assert b"Choice [skip]:" not in output
+        assert not psutil.pid_exists(hook.pid)
+        assert b"Traceback" not in output
+        with use_archivebox_db(initialized_archive):
+            assert not ArchiveResult.objects.filter(hook_name="on_Snapshot__30_chrome_navigate").exists()
+            if plugins == "archivewebpage":
+                start = ArchiveResult.objects.get(hook_name="on_Snapshot__16_archivewebpage_start")
+                assert start.status == ArchiveResult.StatusChoices.NORESULTS
+                assert not ArchiveResult.objects.filter(hook_name="on_Snapshot__65_archivewebpage_stop").exists()
+                assert not ArchiveResult.objects.filter(plugin="archivewebpage", status="succeeded").exists()
+                assert not any(output["name"] == "archivewebpage" for output in start.snapshot.get_html_details_context()["archiveresults"])
+        for log in (initialized_archive / "logs").glob("worker_runner_add_*.log"):
+            assert "Choice [skip]:" not in log.read_text()
+    finally:
+        if result is not None and result.poll() is None:
+            result.terminate()
+            result.wait(timeout=15)
+        os.close(slave)
+        os.close(master)
+
+
+@pytest.mark.parametrize("choice", ["skip", "retry", "ctrl-c", "double", "noninteractive"])
+def test_add_interrupts_background_only_capture(initialized_archive, choice):
+    """Neither pause nor abort/reclaim semantics may depend on a foreground hook."""
+    master, slave = pty.openpty()
+    termios.tcsetwinsize(slave, (40, 200))
+    output = bytearray()
+    result = None
+
+    def read_until(predicate, timeout=45):
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            if select.select([master], [], [], 0.05)[0]:
+                output.extend(os.read(master, 65536))
+            if predicate():
+                return
+        raise AssertionError(output.decode(errors="replace"))
+
+    def active_hook():
+        with use_archivebox_db(initialized_archive):
+            return Process.objects.filter(archiveresult__plugin="forumdl", status="running").order_by("-started_at").first()
+
+    try:
+        result = run_archivebox_cmd(
+            ["add", "--depth=1", "--plugins=forumdl", "https://news.ycombinator.com"],
+            cwd=initialized_archive,
+            env=cli_env(),
+            stdin=subprocess.DEVNULL if choice == "noninteractive" else slave,
+            stdout=slave,
+            stderr=slave,
+            wait=False,
+            start_new_session=True,
+        )
+        read_until(lambda: active_hook() is not None)
+        hook = active_hook()
+        children = psutil.Process(result.pid).children(recursive=True)
+        result.send_signal(signal.SIGINT)
+        if choice == "double":
+            read_until(lambda: b"Forwarding Ctrl+C" in output)
+            result.send_signal(signal.SIGINT)
+        elif choice != "noninteractive":
+            read_until(lambda: b"Choice [skip]:" in output, timeout=20)
+            read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
+            assert not psutil.pid_exists(hook.pid)
+            if choice == "retry":
+                os.write(master, b"r")
+                read_until(lambda: (next_hook := active_hook()) is not None and next_hook.pid != hook.pid)
+                children.extend(psutil.Process(result.pid).children(recursive=True))
+                offset = len(output)
+                result.send_signal(signal.SIGINT)
+                read_until(lambda: b"Choice [skip]:" in output[offset:], timeout=20)
+                read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
+            os.write(master, b"\r" if choice == "skip" else b"\x03")
+        read_until(lambda: result.poll() is not None, timeout=30)
+        assert result.returncode == (0 if choice == "skip" else 130), output.decode(errors="replace")
+        assert not [child.pid for child in children if child.is_running() and child.status() != psutil.STATUS_ZOMBIE]
+        assert not psutil.pid_exists(hook.pid)
+        assert b"Traceback" not in output
+        if choice == "noninteractive":
+            assert b"Choice [skip]:" not in output
+        # An explicit abort must not be mistaken for takeover and restart work.
+        with use_archivebox_db(initialized_archive):
+            assert Process.objects.filter(worker_type="worker_runner", process_type="orchestrator").count() == 1
+        if choice != "skip":
+            assert b"Aborting crawl" in output
+    finally:
+        if result is not None and result.poll() is None:
+            result.terminate()
+            result.wait(timeout=20)
+        os.close(slave)
+        os.close(master)
+
+
+def test_background_completions_render_above_interrupt_prompt(initialized_archive):
+    master, slave = pty.openpty()
+    termios.tcsetwinsize(slave, (40, 200))
+    output = bytearray()
+    result = None
+
+    def read_until(predicate, timeout=45):
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            if select.select([master], [], [], 0.1)[0]:
+                output.extend(os.read(master, 65536))
+            if predicate():
+                return
+        raise AssertionError(output.decode(errors="replace"))
+
+    def result_for(plugin):
+        with use_archivebox_db(initialized_archive):
+            return ArchiveResult.objects.filter(plugin=plugin).first()
+
+    try:
+        result = run_archivebox_cmd(
+            ["add", "--depth=1", "--plugins=wget,infiniscroll,parse_html_urls", "https://example.com"],
+            cwd=initialized_archive,
+            env=cli_env(WGET_ARGS_EXTRA='["--limit-rate=100"]', INFINISCROLL_SCROLL_DELAY="10000", CHROME_HEADLESS="True"),
+            stdin=slave,
+            stdout=slave,
+            stderr=slave,
+            wait=False,
+            start_new_session=True,
+        )
+        read_until(lambda: (scroll := result_for("infiniscroll")) is not None and scroll.status == "started")
+        result.send_signal(signal.SIGINT)
+        read_until(lambda: b"Choice [skip]:" in output)
+        read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
+        before_completion = len(output)
+        read_until(lambda: (wget := result_for("wget")) is not None and wget.status == "succeeded")
+        read_until(lambda: b"Choice [skip]:" in output[before_completion:])
+        screen_updates = re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", output[before_completion:].decode(errors="replace"))
+        completed_at = screen_updates.index("on_Snapshot__35_wget")
+        prompt_at = screen_updates.rindex("Interrupted on_Snapshot__45_infiniscroll")
+        assert "succeeded" in screen_updates[completed_at:prompt_at], screen_updates
+        assert completed_at < prompt_at < screen_updates.rindex("Choice [skip]:"), screen_updates
+        assert result_for("parse_html_urls") is None
+        (initialized_archive / "interrupt-terminal.txt").write_bytes(output)
+        os.write(master, b"a")
+        read_until(lambda: result.poll() is not None)
+        assert result.returncode == 130, output.decode(errors="replace")
+        assert b"Traceback" not in output
+    finally:
+        if result is not None and result.poll() is None:
+            result.terminate()
+            result.wait(timeout=15)
+        os.close(slave)
+        os.close(master)
+
+
+def test_abort_stops_long_running_background_hook(initialized_archive):
+    master, slave = pty.openpty()
+    termios.tcsetwinsize(slave, (40, 200))
+    output = bytearray()
+    result = None
+
+    def read_until(predicate, timeout=45):
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            if select.select([master], [], [], 0.1)[0]:
+                output.extend(os.read(master, 65536))
+            if predicate():
+                return
+        raise AssertionError(output.decode(errors="replace"))
+
+    def result_for(plugin):
+        with use_archivebox_db(initialized_archive):
+            return ArchiveResult.objects.filter(plugin=plugin).first()
+
+    try:
+        result = run_archivebox_cmd(
+            ["add", "--depth=1", "--plugins=forumdl,infiniscroll", "https://news.ycombinator.com"],
+            cwd=initialized_archive,
+            env=cli_env(INFINISCROLL_SCROLL_DELAY="10000", CHROME_HEADLESS="True"),
+            stdin=slave,
+            stdout=slave,
+            stderr=slave,
+            wait=False,
+            start_new_session=True,
+        )
+        read_until(lambda: (scroll := result_for("infiniscroll")) is not None and scroll.status == "started")
+        read_until(lambda: "running pid=" in re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", output.decode(errors="replace")))
+        result.send_signal(signal.SIGINT)
+        read_until(lambda: b"Choice [skip]:" in output)
+        read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
+        forum = result_for("forumdl")
+        assert forum is not None and forum.status == "started"
+        children = psutil.Process(result.pid).children(recursive=True)
+        abort_offset = len(output)
+        os.write(master, b"\x03")
+        read_until(lambda: b"Aborting crawl" in output[abort_offset:], timeout=3)
+        read_until(lambda: result.poll() is not None, timeout=25)
+        assert result.returncode == 130, output.decode(errors="replace")
+        assert not [child.pid for child in children if child.is_running() and child.status() != psutil.STATUS_ZOMBIE]
+        assert any("INFO:root:GET" in log.read_text() for log in initialized_archive.rglob("on_Snapshot__33_forumdl.*.stderr.log"))
+        assert b"Stopped during crawl abort" in output
+        assert b"INFO:root:GET" not in output[abort_offset:]
+        assert b"Traceback" not in output
+    finally:
+        if result is not None and result.poll() is None:
+            result.terminate()
+            result.wait(timeout=15)
+        os.close(slave)
+        os.close(master)
+
+
+@pytest.mark.parametrize("terminal_stream", ["stdout", "stderr"])
+def test_add_renders_live_progress_in_foreground_terminal(initialized_archive, terminal_stream):
+    master, slave = pty.openpty()
+    termios.tcsetwinsize(slave, (40, 160))
+    output = bytearray()
+    result = None
+    try:
+        result = run_archivebox_cmd(
+            ["add", "--plugins=parse_txt_urls", "https://example.com"],
+            cwd=initialized_archive,
+            env=cli_env(USE_COLOR="True", SHOW_PROGRESS="True"),
+            **{terminal_stream: slave},
+            wait=False,
+            start_new_session=True,
+        )
+        deadline = time.monotonic() + 90
+        while time.monotonic() < deadline:
+            if select.select([master], [], [], 0.1)[0]:
+                output.extend(os.read(master, 65536))
+            elif result.poll() is not None:
+                break
+        assert result.poll() == 0, output.decode(errors="replace")
+        assert b"\x1b[?25l" in output, output.decode(errors="replace")
+        assert b"\x1b[?25h" in output, output.decode(errors="replace")
+        assert b"on_Snapshot__71_parse_txt_urls" in output
+        assert b"\x1b[2K" in output
+        with use_archivebox_db(initialized_archive):
+            assert Crawl.objects.get().status == Crawl.StatusChoices.SEALED
+            snapshot = Snapshot.objects.get()
+            assert snapshot.url == "https://example.com"
+            parsed_urls = Path(snapshot.output_dir) / "parse_txt_urls" / "urls.jsonl"
+            assert json.loads(parsed_urls.read_text())["url"] == "https://iana.org/domains/example"
+    finally:
+        if result is not None and result.poll() is None:
+            result.terminate()
+            result.wait(timeout=10)
+        os.close(slave)
+        os.close(master)
+
+
+def test_add_redirected_progress_remains_plain_text(initialized_archive):
+    result = run_archivebox_cmd(
+        ["add", "--plugins=parse_txt_urls", "https://example.com"],
+        cwd=initialized_archive,
+        env=cli_env(),
+    )
+    assert result.returncode == 0, result.stderr
+    assert "on_Snapshot__71_parse_txt_urls" in result.stderr
+    assert "\x1b[?25l" not in result.stdout + result.stderr
+
+
+def test_add_summary_points_to_snapshot_outputs(initialized_archive):
+    result = run_archivebox_cmd(
+        ["add", "--plugins=parse_txt_urls", "https://example.com"],
+        cwd=initialized_archive,
+        env=cli_env(),
+    )
+    assert result.returncode == 0, result.stderr
+    with use_archivebox_db(initialized_archive):
+        snapshot = Snapshot.objects.get()
+        assert Path(snapshot.output_dir).is_dir()
+        assert str(snapshot.output_dir) in result.stdout
+        assert "snapshot output saved to:" in result.stdout
+        assert "crawl logs and setup:" in result.stdout
+        assert "crawl output saved to:" not in result.stdout
+
+
 IMPORT_FORMAT_EXPECTATIONS = {
     "rss": {
         "url": "https://example.com/",
@@ -1204,7 +1552,7 @@ def test_cli_recursive_crawl_processes_discovered_html_urls(initialized_archive,
     root_url = recursive_test_site["root_url"]
     child_url = recursive_test_site["child_urls"][0]
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         [
             "add",
             "--depth=2",
@@ -1219,8 +1567,7 @@ def test_cli_recursive_crawl_processes_discovered_html_urls(initialized_archive,
         env=env,
         timeout=180,
     )
-    stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-    assert returncode == 0, stderr or stdout
+    assert result.returncode == 0, result.stderr or result.stdout
 
     with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first()

archivebox/tests/test_cli_archiveresult.py

@@ -142,18 +142,17 @@ class TestArchiveResultCreate:
         assert crawl_result.returncode == 0, crawl_result.stderr
         crawl_record = parse_jsonl_output(crawl_result.stdout)[0]
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["archiveresult", "create"],
             stdin=crawl_result.stdout,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        assert "Passed through" in stderr
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        assert "Passed through" in result.stderr
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 1
         assert records[0]["id"] == crawl_record["id"]
 
@@ -163,16 +162,15 @@ class TestArchiveResultList:
 
     def test_list_empty(self, initialized_archive):
         """List with no archive results returns empty."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["archiveresult", "list"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        assert "Listed 0 archive results" in stderr
+        assert result.returncode == 0
+        assert "Listed 0 archive results" in result.stderr
 
     def test_list_filter_by_status(self, initialized_archive):
         """Filter archive results by status."""

archivebox/tests/test_cli_crawl.py

@@ -33,19 +33,18 @@ class TestCrawlCreate:
         """Create crawl from URL arguments."""
         url = create_test_url()
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create", url],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, f"Command failed: {stderr}"
-        assert "Created crawl" in stderr
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+        assert "Created crawl" in result.stderr
 
         # Check JSONL output
-        records = parse_jsonl_output(stdout)
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 1
         assert records[0]["type"] == "Crawl"
         assert url in records[0]["urls"]
@@ -55,18 +54,17 @@ class TestCrawlCreate:
         urls = [create_test_url() for _ in range(3)]
         stdin = "\n".join(urls)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create"],
             stdin=stdin,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, f"Command failed: {stderr}"
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
 
-        records = parse_jsonl_output(stdout)
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 1
         crawl = records[0]
         assert crawl["type"] == "Crawl"
@@ -78,32 +76,30 @@ class TestCrawlCreate:
         """Create crawl with --depth flag."""
         url = create_test_url()
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create", "--depth=2", url],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert records[0]["max_depth"] == 2
 
     def test_create_with_tag(self, initialized_archive):
         """Create crawl with --tag flag."""
         url = create_test_url()
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create", "--tag=test-tag", url],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert "test-tag" in records[0].get("tags_str", "")
 
     def test_create_passes_through_tag_emitted_by_cli(self, initialized_archive):
@@ -119,17 +115,16 @@ class TestCrawlCreate:
         url = create_test_url()
         stdin = tag_result.stdout + json.dumps({"url": url}) + "\n"
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create"],
             stdin=stdin,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
 
         assert any(record.get("type") == "Tag" and record["id"] == tag_record["id"] for record in records)
         assert any(record.get("type") == "Crawl" and url in record["urls"] for record in records)
@@ -163,32 +158,30 @@ class TestCrawlList:
 
     def test_list_empty(self, initialized_archive):
         """List with no crawls returns empty."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "list"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        assert "Listed 0 crawls" in stderr
+        assert result.returncode == 0
+        assert "Listed 0 crawls" in result.stderr
 
     def test_list_returns_created(self, initialized_archive):
         """List returns previously created crawls."""
         url = create_test_url()
         run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "list"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) >= 1
         assert any(url in r.get("urls", "") for r in records)
 
@@ -197,16 +190,15 @@ class TestCrawlList:
         url = create_test_url()
         run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "list", "--status=queued"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         for r in records:
             assert r["status"] == "queued"
 
@@ -221,16 +213,15 @@ class TestCrawlList:
                 disable_extractors=True,
             )
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "list", "--limit=2"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 2