Bump release version to 0.9.55rc5

# Conflicts:
#	etc/package.json
#	pyproject.toml
#	uv.lock

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -30,8 +30,8 @@ body:
       description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
       render: markdown
       placeholder: |
-        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox` in iTerm2
-        2. Went to the https://127.0.0.1:8000/add/ page in Google Chrome
+        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 5797:5797 archivebox/archivebox` in iTerm2
+        2. Went to the https://127.0.0.1:5797/add/ page in Google Chrome
         3. Typed 'https://example.com/xyz' into the 'Add URL' input field
         4. Clicked the 'Add+' button
         5. Got a 500 error and saw the errors below in terminal
@@ -138,7 +138,7 @@ body:
             archivebox:
                 image: archivebox/archivebox:latest
                 ports:
-                    - 8000:8000
+                    - 5797:5797
                 volumes:
                     - ./data:/data
                 environment:

.github/configs/ci-tooling.json

@@ -116,6 +116,10 @@
     }
   ],
   "docker_binaries": [
+    {
+      "name": "{BASH_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
     {
       "name": "{DOCKER_BINARY}",
       "binproviders": "env"

.github/pages/README.md

@@ -0,0 +1,119 @@
+# archivebox.io website
+
+This repository owns its complete website. Build and deploy it without checking
+out another repository's website, fetching shared snippets, or calling a shared
+workflow. Common files are ordinary committed copies; update them deliberately.
+
+## Repository-specific files
+
+- Content: `publicsite/index.html`.
+- Site/gallery generation: `bin/generate_ui_screenshot_gallery.py`.
+- Application capture: `bin/collect_ui_screenshots.sh`.
+- Deployment branch: `dev`.
+
+## Common publication contract
+
+1. Build pull requests without deploying them.
+2. Build website pushes immediately, independently of application capture CI.
+3. Restore validated screenshots from trusted default-branch capture jobs.
+   Keep the original capture revision, timestamp, coverage and image checksums.
+4. Retain `site-screenshots` artifacts for 90 days. After expiry, restore the
+   published image set. Never replace a complete gallery with an empty one when
+   restoration fails. Apple device groups are independently validated artifacts;
+   Android's checked-in bootstrap captures retain their local provenance.
+5. Render current content and layout around those captures. Site revision and
+   capture revision are separate facts; reusing captures must not relabel them.
+6. Serialize production restore/build/deploy under one concurrency group and
+   check out the latest default-branch revision when each build starts. Keep the
+   built revision in provenance; later release-bot commits must not block deploys.
+7. Keep platform-specific build, installation, capture, and completeness checks
+   in the application's existing capture workflow. Site-only edits do not need
+   to rerun those jobs.
+
+## Common presentation contract
+
+The base header, Apps menu and icons, footer, brand styles and navigation behavior
+should remain identical copies across the seven product sites. Put this site's
+name, navigation, CTA and extra footer resources outside the common base files.
+Preserve real links, anchor IDs, no-JavaScript navigation, keyboard operation,
+mobile layout, reduced-motion preferences and existing product-specific content.
+Screenshots stay in English. Translation content, when added, lives in separate
+HTML files; no runtime translation service or cross-repository dependency.
+
+## Editing and building
+
+`base/` contains identical independent copies across the seven product sites.
+Common Python scripts share the same behavior; repository formatters may change
+line wrapping without changing their syntax trees. The base owns the shared logo, Apps menu/icons, footer directory, navigation
+behavior and styles. `site.json` and `nav.html` contain this site's name, routes,
+links and CTA. Product content and additional footer resources stay in the native
+HTML/Jinja/Liquid templates listed above. The `ARCHIVEBOX:*` comments are build
+slots; the renderer fills them with static HTML, never browser-side fetches.
+
+The existing builder calls `site.py render` after generating its content. This
+copies `base/chrome.css` and `base/chrome.js` to `site-base/`, assembles the common
+HTML, validates local resources and writes the site revision to root `build.json`.
+Screenshot manifests are left untouched. No build reads another repository.
+
+Common changes are manual copy/paste edits. Compare `base/` and `site.py` with
+another product repository when updating them; do not add a synchronization job,
+package dependency or reusable cross-repository workflow. `nav.html`, `site.json`
+and the marked workflow build section are intentionally repository-specific.
+
+Build: `uv run --no-project python .github/pages/site.py render _site --source publicsite` after restoring screenshots and refreshing the gallery.
+
+Check the built site with real Chromium (also run by Pages CI):
+
+```sh
+uv run --no-config --no-project --with playwright==1.63.0 playwright install chromium
+uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py SITE_OUTPUT --evidence /tmp/site-evidence
+```
+
+Use this workflow's `SITE_OUTPUT` directory. The check covers desktop/mobile
+layout, keyboard dismissal, no-JavaScript links, local resources and visible
+images, and uploads its screenshots as `site-verification` for 14 days.
+
+## Branding and future translations
+
+`base/chrome.css` owns the shared system font stacks, plum accent and CTA shape,
+including the existing `.button` and `.ab-btn` classes. Product styles own their
+layouts, feature sections and platform-specific button states. Shared system
+fonts need no external font downloads. `site.py` emits the same complete SEO/OG/
+Twitter tags everywhere while retaining each native template's title, description,
+canonical URL, locale and social artwork; edit those values with that page's copy.
+
+For translations, keep complete separate HTML files under `es/`, `fr/` and `zh/`,
+and list their output paths in `site.json` alongside the English pages. Share CSS,
+JS, social artwork and the English screenshot gallery. Translate each page's title
+and description, set its own canonical and `lang` (`zh` for Chinese), and add
+reciprocal `hreflang` links only for translations that actually exist. The renderer
+preserves these tags and computes shared asset paths for nested pages. Localized
+header/footer copy and the small language selector can be added with those pages;
+there is no language redirect, placeholder translation or translation runtime now.
+
+## Screenshot strips
+
+`marquee.py` and `base/marquee.css` / `base/marquee.js` are also identical copies.
+Product homepages choose their placement with an `ARCHIVEBOX:MARQUEE` comment.
+The renderer takes real images, captions, dimensions and links from the generated
+`screenshots/index.html`; there is no second screenshot list or browser fetch.
+Desktop/tablet/mobile galleries contribute their desktop views. Apple uses
+`MARQUEE-CLIENT` and `MARQUEE-SERVER` slots to keep its two products separate.
+Catalog sites without a slot do not load the strip assets.
+
+The strip scrolls once, pauses on hover, focus or manual interaction, and has an
+explicit Play/Pause button. Reduced-motion users start paused. Without JavaScript,
+normal image links and horizontal scrolling remain available. Screenshot capture
+manifests and provenance are never rewritten by this presentation step.
+
+Standalone one- or two-image illustrations remain in their authored source.
+Gallery strips with three or more images use the latest restored captures.
+
+## Keeping the common layout in sync
+
+The comments in `base/header.html`, `base/footer-start.html`,
+`base/footer-end.html`, `base/chrome.css`, and `base/chrome.js` list the peer
+repositories. Update those committed copies together when shared branding,
+Apps links, keyboard behavior, or footer resources change. Preserve each site's
+local navigation and content. The former monorepo `site-chrome` generator is
+retired; builds must use only this repository's files.

.github/pages/artifacts.py

@@ -0,0 +1,102 @@
+"""Common screenshot artifact transport; coverage checks belong to each app."""
+
+import json
+import subprocess
+from pathlib import PurePosixPath
+from urllib.request import urlopen
+
+
+def api(repo, path):
+    return json.loads(subprocess.check_output(["gh", "api", f"repos/{repo}/{path}"]))
+
+
+def trusted(run, repo, workflow, branch, successful=True):
+    return (
+        (run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
+        and run["head_branch"] == branch
+        and run["event"] in ("push", "workflow_dispatch")
+        and run["path"] == f".github/workflows/{workflow}"
+        and (not successful or run["conclusion"] == "success")
+    )
+
+
+def runs(repo, workflow, branch, successful=True, artifact_names=("site-screenshots",)):
+    # Ask for capture artifacts directly instead of checking every historical
+    # workflow run (most app runs never produce a complete screenshot set).
+    # Artifact creation time also includes fresh captures from old rerun jobs.
+    candidates = []
+    for name in artifact_names:
+        rows = subprocess.check_output(
+            [
+                "gh",
+                "api",
+                f"repos/{repo}/actions/artifacts?name={name}&per_page=100",
+                "--paginate",
+                "--jq",
+                ".artifacts[] | select(.expired == false) | @json",
+            ],
+            text=True,
+        )
+        candidates.extend(json.loads(row) for row in rows.splitlines())
+    seen = set()
+    for artifact in sorted(candidates, key=lambda item: item["created_at"], reverse=True):
+        source = artifact.get("workflow_run") or {}
+        run_id = source.get("id")
+        if (
+            artifact["name"] not in artifact_names
+            or not run_id
+            or run_id in seen
+            or source.get("head_branch") != branch
+            or source.get("head_repository_id") != source.get("repository_id")
+        ):
+            continue
+        seen.add(run_id)
+        run = api(repo, f"actions/runs/{run_id}")
+        if trusted(run, repo, workflow, branch, successful):
+            yield run
+
+
+def names(repo, run):
+    rows = subprocess.check_output(
+        [
+            "gh",
+            "api",
+            f"repos/{repo}/actions/runs/{run['id']}/artifacts?per_page=100",
+            "--paginate",
+            "--jq",
+            ".artifacts[] | @json",
+        ],
+        text=True,
+    )
+    return {artifact["name"] for row in rows.splitlines() if not (artifact := json.loads(row))["expired"]}
+
+
+def download(repo, run, name, destination):
+    subprocess.run(
+        [
+            "gh",
+            "run",
+            "download",
+            str(run["id"]),
+            "--repo",
+            repo,
+            "--name",
+            name,
+            "--dir",
+            str(destination),
+        ],
+        check=True,
+    )
+
+
+def relative_path(name):
+    path = PurePosixPath(name)
+    if not name or path.is_absolute() or ".." in path.parts or any(c in name for c in ":\\?#%"):
+        raise ValueError(f"Unsafe screenshot path: {name}")
+    return path
+
+
+def fetch(base, name):
+    relative_path(name)
+    with urlopen(base + name, timeout=60) as response:
+        return response.read()

.github/pages/base/chrome.css

@@ -0,0 +1,156 @@
+/*
+Shared ArchiveBox website navigation styles and behavior.
+Keep this file in sync with .github/pages/base/chrome.css in:
+ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
+electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
+Each site builds its committed copy independently; there is no central generator.
+Keep site-specific navigation, CTAs, and extra footer links outside this base.
+Also review the inline navbar/footer in abx-dl/website/index.html and
+ debian-archivebox/website/index.html when shared links or branding change.
+*/
+/* Common brand tokens. Native templates own product layout and content. */
+:root {
+  --abx-font-sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+  --abx-font-mono: ui-monospace, "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace;
+  --abx-accent: #9b2854;
+  --abx-accent-hover: #782040;
+  --abx-paper: #fcfaf7;
+  --abx-ink: #28252a;
+  --abx-muted: #716c74;
+  --abx-line: #e9e2e3;
+  --font-sans: var(--abx-font-sans);
+  --font-mono: var(--abx-font-mono);
+  --ab-font-sans: var(--abx-font-sans);
+  --ab-font-serif: var(--abx-font-sans);
+  --ab-font-label: var(--abx-font-sans);
+  --ab-font-mono: var(--abx-font-mono);
+  --ab-accent: var(--abx-accent);
+  --ab-accent-hover: var(--abx-accent-hover);
+  --ab-on-accent: #fff;
+}
+body { font-family: var(--abx-font-sans); }
+code, pre, kbd, samp { font-family: var(--abx-font-mono); }
+/* Reuse existing CTA classes; platform selection continues to control colors. */
+.button, .ab-btn {
+  display: inline-flex; align-items: center; justify-content: center; gap: 10px;
+  min-height: 44px; height: auto; max-width: 100%; padding: 10px 20px;
+  border-radius: 24px; font-family: var(--abx-font-sans);
+  font-size: 14px; font-weight: 650; line-height: 1.4;
+  text-align: center; white-space: normal; text-decoration: none;
+}
+.button.primary, .ab-btn--primary { background: var(--abx-accent); border-color: var(--abx-accent); color: #fff; }
+.button.primary:hover, .ab-btn--primary:hover { background: var(--abx-accent-hover); border-color: var(--abx-accent-hover); color: #fff; }
+.button:focus-visible, .ab-btn:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
+.button svg, .button img, .ab-btn svg, .ab-btn img { flex-shrink: 0; }
+@media (prefers-reduced-motion: reduce) { .button, .ab-btn { transition: none; } .button:hover { transform: none; } }
+
+/* Shared ArchiveBox site chrome. Independent copy; edit alongside the other product sites, without a build dependency. */
+/* Native anchor navigation leaves room for the heading and its surrounding content. */
+html { scroll-behavior: smooth; scroll-padding-top: clamp(96px, 25vh, 240px); }
+@media (prefers-reduced-motion: reduce) { html { scroll-behavior: auto; } }
+.abx-header, .abx-footer { box-sizing: border-box; font-family: var(--abx-font-sans); font-size: 14px; line-height: 1.5; letter-spacing: normal; text-align: left; }
+.abx-header *, .abx-footer * { box-sizing: border-box; }
+.abx-header { position: sticky; top: 0; z-index: 100; display: flex; align-items: center; justify-content: space-between; gap: 24px; width: 100%; max-width: none; margin: 0; padding: 15px max(24px, calc((100% - 1280px) / 2)); border: 0; border-bottom: 1px solid #e9e2e3; background: #fcfaf7f5; color: #28252a; backdrop-filter: blur(16px); }
+.abx-header a, .abx-footer a { text-decoration: none; text-underline-offset: .24em; }
+.abx-header .abx-brand, .abx-footer .abx-brand { display: inline-flex; flex: 0 0 auto; align-items: center; gap: 11px; margin: 0; padding: 0; color: inherit; font-size: 19px; font-weight: 750; line-height: 1.2; letter-spacing: -.5px; white-space: nowrap; }
+.abx-brand .abx-logo { display: block; flex: 0 0 auto; width: 32px; height: 32px; margin: 0; }
+.abx-header .abx-subsite { transform: translateY(3px); margin-left: 2px; color: #716c74; font-size: 12px; font-weight: 450; letter-spacing: 0; }
+.abx-header .abx-nav { display: flex; align-items: center; justify-content: flex-end; flex-wrap: wrap; gap: 0; width: auto; max-width: none; margin: 0; padding: 0; font-size: 13px; font-weight: 600; }
+.abx-nav > a, .abx-nav > details { margin: 0 0 0 15px; padding: 0 0 0 15px; border: 0; border-left: 1px solid #ded4d8; border-radius: 0; background: none; }
+.abx-nav > :first-child { margin-left: 0; padding-left: 0; border-left: 0; }
+.abx-header .abx-nav a { display: inline-flex; align-items: center; min-height: 36px; color: #4d424b; white-space: nowrap; }
+.abx-header .abx-nav a:hover { color: #9b2854; }
+.abx-header .abx-nav .abx-cta { margin-left: 18px; padding: 9px 17px; border: 1px solid #9b2854; border-radius: 24px; background: #9b2854; color: #fff; }
+.abx-header .abx-nav .abx-cta:hover { background: #782040; color: #fff; }
+.abx-header .abx-apps { position: relative; }
+.abx-apps > summary { display: flex; align-items: center; gap: 7px; min-height: 36px; list-style: none; cursor: pointer; color: #4d424b; }
+.abx-apps > summary::-webkit-details-marker { display: none; }
+.abx-apps > summary::before { content: '›'; display: inline-block; color: #9b2854; font-size: 22px; line-height: 1; transition: transform .15s; }
+.abx-apps[open] > summary::before { transform: rotate(90deg); }
+.abx-header .abx-app-links { position: absolute; right: 0; top: calc(100% + 12px); width: 440px; max-width: calc(100vw - 24px); max-height: min(70vh, 580px); overflow-y: auto; padding: 8px; border: 1px solid #e9e2e3; border-radius: 16px; background: #fcfaf7; box-shadow: 0 18px 48px #2d152522, 0 3px 8px #2d152508; }
+.abx-header .abx-app-row { display: flex; align-items: center; gap: 8px; min-height: 44px; padding: 4px 6px; border-radius: 10px; transition: background .15s; }
+.abx-header .abx-app-row:hover, .abx-header .abx-app-row:focus-within { background: #f2e8ed; }
+.abx-header .abx-app-links .abx-app-name { flex: 1; min-width: 0; min-height: 34px; padding: 5px 4px; border-radius: 6px; font-size: 13px; line-height: 1.35; white-space: normal; }
+.abx-header .abx-platforms { display: flex; flex: 0 0 auto; align-items: center; gap: 5px; }
+.abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 auto; height: 30px; min-height: 30px; padding: 5px; border: 1px solid #e8dfe3; border-radius: 9px; background: #fffdfb; color: #4d424b; box-shadow: 0 2px 3px #2d152510, inset 0 1px 0 #fff; transition: transform .15s, box-shadow .15s, border-color .15s; }
+.abx-header .abx-app-links .abx-platform { width: 30px; }
+.abx-header .abx-platform svg { display: block; width: 18px; height: 18px; margin: 0; fill: currentColor; }
+.abx-header .abx-app-links .abx-platform-docker { color: #2386ce; }
+.abx-header .abx-app-links .abx-platform-windows, .abx-header .abx-app-links .abx-platform-app-store, .abx-header .abx-app-links .abx-platform-safari { color: #1879c4; }
+.abx-header .abx-app-links .abx-platform-android { color: #29965d; }
+.abx-header .abx-app-links .abx-platform-firefox { color: #e46b24; }
+.abx-header .abx-app-links .abx-platform-brave { color: #e8552b; }
+.abx-header .abx-app-links .abx-platform-edge { color: #168b95; }
+.abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-platform:focus-visible, .abx-header .abx-app-links .abx-tool:hover, .abx-header .abx-app-links .abx-tool:focus-visible { transform: translateY(-2px); border-color: #c99bb0; background: #fff; box-shadow: 0 5px 10px #2d15251c; }
+.abx-header .abx-app-links .abx-tool { padding: 5px 8px; font-size: 11px; white-space: nowrap; }
+.abx-header .abx-tools-row { margin-top: 5px; padding-top: 9px; border-top: 1px solid #e9e2e3; border-radius: 0 0 10px 10px; }
+.abx-header .abx-tools-row .abx-app-name { flex: 0 0 auto; }
+.abx-header .abx-tools-row .abx-platforms { flex: 0 1 auto; flex-wrap: wrap; justify-content: flex-end; margin-left: auto; }
+.abx-header .abx-app-links code { display: inline; vertical-align: baseline; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-size: 1em; font-weight: inherit; line-height: inherit; margin: 0; padding: 0; border: 0; background: transparent; color: inherit; }
+.abx-header .abx-icon-stack { isolation: isolate; gap: 0; padding: 4px 5px; }
+.abx-header .abx-icon-stack .abx-integration { position: relative; z-index: 1; transform: translateY(var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); transition: transform .18s ease, box-shadow .18s ease, border-color .18s ease; }
+.abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -10px; }
+.abx-header .abx-icon-stack .abx-integration:nth-child(2) { --stack-angle: 9deg; --stack-y: -2px; z-index: 2; color: #569a31; }
+.abx-header .abx-icon-stack .abx-integration:nth-child(3) { --stack-angle: -6deg; --stack-y: 2px; z-index: 3; }
+.abx-header .abx-icon-stack .abx-integration:nth-child(4) { --stack-angle: 7deg; --stack-y: -1px; z-index: 4; color: #f00; }
+.abx-header .abx-icon-stack .abx-integration:nth-child(5) { --stack-angle: -10deg; --stack-y: 1px; z-index: 5; }
+.abx-header .abx-icon-stack .abx-integration:nth-child(6) { --stack-angle: 6deg; --stack-y: -2px; z-index: 6; }
+.abx-header .abx-icon-stack .abx-integration:nth-child(7) { --stack-angle: -5deg; z-index: 7; }
+.abx-header .abx-icon-stack .abx-integration:not(:hover, :focus-visible):has(~ .abx-integration:is(:hover, :focus-visible)) { transform: translate(-6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
+.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) ~ .abx-integration:not(:hover, :focus-visible) { transform: translate(6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
+.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { z-index: 20; transform: translateY(-7px) rotate(0deg) scale(1.12); border-color: #c99bb0; box-shadow: 0 8px 14px #2d15252b; }
+@media (hover: none) { .abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -6px; } }
+@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration { transition: none; } }
+.abx-footer { position: relative; z-index: 1; display: block; width: 100%; max-width: none; margin: 72px 0 0; padding: 60px max(24px, calc((100% - 1280px) / 2)) 24px; border: 0; border-top: 3px solid #7e284a; background: radial-gradient(ellipse at 12% 0, #54243b55, transparent 48%), linear-gradient(120deg, #201923, #14151b 75%); color: #ddd2dd; color-scheme: dark; }
+.abx-footer .abx-footer-intro { display: flex; align-items: flex-start; justify-content: space-between; gap: 32px; margin: 0 0 42px; }
+.abx-footer .abx-brand { color: #fff4ed; font-size: 24px; }
+.abx-footer .abx-brand .abx-logo { width: 40px; height: 40px; }
+.abx-footer .abx-tagline { max-width: 430px; margin: 12px 0 0; color: #b8acbb; font-size: 14px; line-height: 1.7; }
+.abx-footer .abx-footer-motto { margin: 2px 0 0; color: #d8b39f; font: italic 25px/1.45 Georgia, serif; text-align: right; }
+.abx-footer .abx-footer-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 42px; margin: 0; }
+.abx-footer .abx-footer-column { margin: 0; padding: 20px 0 0; border: 0; border-top: 1px solid #67505e66; background: none; }
+.abx-footer .abx-footer-column h2 { margin: 0 0 19px; padding: 0; border: 0; color: #e4acbb; font: 650 12px/1.4 var(--abx-font-sans); letter-spacing: .12em; text-transform: uppercase; }
+.abx-footer .abx-footer-column ul { display: block; list-style: none; margin: 0; padding: 0; }
+.abx-footer .abx-footer-column li { display: flex; align-items: center; gap: 10px; margin: 0; padding: 0; min-height: 36px; }
+.abx-footer .abx-footer-column li::before { content: none; }
+.abx-footer .abx-footer-column a { color: #ddd2dd; font-size: 13px; line-height: 1.55; }
+.abx-footer .abx-footer-column a:first-child { padding: 6px 0; }
+.abx-footer a:hover { color: #fff; text-decoration: underline; }
+.abx-footer .abx-source { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 28px; width: 28px; height: 28px; border: 1px solid transparent; border-radius: 50%; color: #a58d9f; }
+.abx-footer .abx-source:hover { color: #f3b5cb; border-color: #80516a; background: #ffffff08; }
+.abx-source svg { width: 15px; height: 15px; fill: currentColor; }
+.abx-footer .abx-footer-bottom { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px 24px; margin-top: 32px; padding-top: 21px; border-top: 1px solid #67505e66; color: #a99bab; font-size: 12px; }
+.abx-footer .abx-footer-bottom p { margin: 0; font-size: inherit; color: inherit; }
+.abx-footer .abx-footer-bottom a { color: #cebbc9; }
+.abx-footer .abx-footer-local { margin: 24px 0 0; padding: 16px 0 0; border: 0; border-top: 1px solid #67505e66; border-radius: 0; background: none; color: #baacbc; font-size: 12px; }
+.abx-footer-local > summary { cursor: pointer; color: #d6bfce; }
+.abx-footer-local > div { padding-top: 18px; }
+.abx-footer-local a { color: #e5aabd; }
+.abx-footer-local .brand, .abx-footer-local .footer-intro, .abx-footer-local .footer-grid, .abx-footer-local .footer-meta { color: inherit; }
+.abx-footer-local .footer-card { background: transparent; border-color: #67505e66; }
+.abx-footer-local h2, .abx-footer-local h3, .abx-footer-local strong { color: #e4d8e3; }
+.abx-footer-local .footer-link-copy span, .abx-footer-local .footer-card-subtitle { color: #baacbc; }
+.abx-header a:focus-visible, .abx-header summary:focus-visible, .abx-footer a:focus-visible, .abx-footer summary:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
+@media (max-width: 1100px) {
+  .abx-header { flex-wrap: wrap; gap: 8px 24px; padding: 12px 20px; }
+  .abx-header .abx-nav { justify-content: flex-start; }
+  .abx-header .abx-app-links { position: fixed; top: auto; left: 20px; right: auto; width: 440px; margin-top: 10px; }
+}
+@media (max-width: 700px) {
+  .abx-header { position: relative; padding: 14px 18px; }
+  .abx-header .abx-brand { flex-wrap: wrap; row-gap: 5px; }
+  .abx-header .abx-nav { width: 100%; gap: 4px 0; font-size: 12px; }
+  .abx-nav > a, .abx-nav > details { margin-left: 10px; padding-left: 10px; }
+  .abx-header .abx-nav .abx-cta { margin-left: 10px; padding: 7px 13px; }
+  .abx-header .abx-app-links { left: 12px; right: 12px; width: auto; max-height: 60vh; }
+  .abx-footer { padding: 40px 22px 22px; margin-top: 48px; }
+  .abx-footer .abx-footer-intro { display: block; margin-bottom: 30px; }
+  .abx-footer .abx-footer-motto { margin-top: 20px; text-align: left; font-size: 22px; }
+  .abx-footer .abx-footer-grid { grid-template-columns: 1fr; gap: 26px; }
+  .abx-footer .abx-footer-column li { min-height: 40px; }
+}
+@media (prefers-reduced-motion: reduce) { .abx-apps > summary::before, .abx-header .abx-app-row, .abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { transition: none; } .abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-tool:hover { transform: none; } }
+
+.abx-footer-local > div > a + a { margin-left: 12px; }
+
+@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { transform: none; } }

.github/pages/base/chrome.js

@@ -0,0 +1,28 @@
+/*
+Shared ArchiveBox website navigation styles and behavior.
+Keep this file in sync with .github/pages/base/chrome.js in:
+ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
+electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
+Each site builds its committed copy independently; there is no central generator.
+Keep site-specific navigation, CTAs, and extra footer links outside this base.
+Also review the inline navbar/footer in abx-dl/website/index.html and
+ debian-archivebox/website/index.html when shared links or branding change.
+*/
+(() => {
+  const menu = document.querySelector('.abx-apps');
+  document.addEventListener('keydown', event => {
+    if (event.key === 'Escape' && menu?.open) { menu.open = false; menu.querySelector('summary').focus(); }
+  });
+  document.addEventListener('click', event => { if (menu?.open && !menu.contains(event.target)) menu.open = false; });
+  function revealAnchor() {
+    let target;
+    try { target = document.getElementById(decodeURIComponent(window.location.hash.slice(1))); } catch { return; }
+    if (!target) return;
+    let details = target.closest('details');
+    let opened = false;
+    while (details) { if (!details.open) { details.open = true; opened = true; } details = details.parentElement.closest('details'); }
+    if (opened) window.requestAnimationFrame(() => target.scrollIntoView());
+  }
+  window.addEventListener('hashchange', revealAnchor);
+  revealAnchor();
+})();

.github/pages/base/footer-end.html

@@ -0,0 +1,12 @@
+<!--
+Shared ArchiveBox website footer.
+Keep this file in sync with .github/pages/base/footer-end.html in:
+ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
+electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
+Each site builds its committed copy independently; there is no central generator.
+Keep site-specific navigation, CTAs, and extra footer links outside this base.
+Also review the inline navbar/footer in abx-dl/website/index.html and
+ debian-archivebox/website/index.html when shared links or branding change.
+-->
+  <div class="abx-footer-bottom"><p>Built in the open. Supported by our community.</p><p><a href="https://github.com/ArchiveBox">GitHub organization</a> · <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Security &amp; privacy</a> · <a href="https://archivebox.io">ArchiveBox.io ↗</a></p></div>
+</footer>

.github/pages/base/footer-start.html

@@ -0,0 +1,13 @@
+<!--
+Shared ArchiveBox website footer.
+Keep this file in sync with .github/pages/base/footer-start.html in:
+ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
+electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
+Each site builds its committed copy independently; there is no central generator.
+Keep site-specific navigation, CTAs, and extra footer links outside this base.
+Also review the inline navbar/footer in abx-dl/website/index.html and
+ debian-archivebox/website/index.html when shared links or branding change.
+-->
+<footer class="abx-footer" aria-label="ArchiveBox ecosystem">
+  <div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
+  <div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps &amp; tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS &amp; iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS &amp; iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated

.github/pages/base/header.html

@@ -0,0 +1,24 @@
+<!--
+Shared ArchiveBox website navbar and Apps menu.
+Keep this file in sync with .github/pages/base/header.html in:
+ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
+electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
+Each site builds its committed copy independently; there is no central generator.
+Keep site-specific navigation, CTAs, and extra footer links outside this base.
+Also review the inline navbar/footer in abx-dl/website/index.html and
+ debian-archivebox/website/index.html when shared links or branding change.
+-->
+<header class="abx-header">
+  <a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span><span class="abx-subsite">__SITE_NAME__</span></a>
+  <nav class="abx-nav" aria-label="Main navigation">
+    <details class="abx-apps"><summary>Apps</summary><div class="abx-app-links">
+      <div class="abx-app-row"><a class="abx-app-name" href="https://archivebox.io/#install-compose">ArchiveBox Server</a><div class="abx-platforms"><a class="abx-platform abx-platform-docker" href="https://archivebox.io/#install-compose" aria-label="ArchiveBox Server for Docker" title="ArchiveBox Server for Docker"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M349.9 236.3l-66.1 0 0-59.4 66.1 0 0 59.4zm0-204.3l-66.1 0 0 60.7 66.1 0 0-60.7zm78.2 144.8l-66.1 0 0 59.4 66.1 0 0-59.4zM271.8 104.7l-66.1 0 0 60.1 66.1 0 0-60.1zm78.1 0l-66.1 0 0 60.1 66.1 0 0-60.1zm276.8 100c-14.4-9.7-47.6-13.2-73.1-8.4-3.3-24-16.7-44.9-41.1-63.7l-14-9.3-9.3 14c-18.4 27.8-23.4 73.6-3.7 103.8-8.7 4.7-25.8 11.1-48.4 10.7l-434.7 0c-8.7 50.8 5.8 116.8 44 162.1 37.1 43.9 92.7 66.2 165.4 66.2 157.4 0 273.9-72.5 328.4-204.2 21.4 .4 67.6 .1 91.3-45.2 1.5-2.5 6.6-13.2 8.5-17.1l-13.3-8.9zM115.6 176.8l-66 0 0 59.4 66.1 0 0-59.4-.1 0zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm-78.1-72.1l-66.1 0 0 60.1 66.1 0 0-60.1z"/></svg></a><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/#archivebox-serverapp" aria-label="ArchiveBox Server for macOS" title="ArchiveBox Server for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://electron.archivebox.io/" aria-label="ArchiveBox Server for Windows" title="ArchiveBox Server for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://archivebox.io/#install-apt" aria-label="ArchiveBox Server for Linux" title="ArchiveBox Server for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-1Line truncated
+      <div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Desktop</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Desktop for macOS" title="ArchiveBox Desktop for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://windows.archivebox.io/" aria-label="ArchiveBox Desktop for Windows" title="ArchiveBox Desktop for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://linux.archivebox.io/" aria-label="ArchiveBox Desktop for Linux" title="ArchiveBox Desktop for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-10 14.1-10.5 4.7-4.4 13.5-14.2 27.9-14.2 7.1 0 15.6 2.3 25.9 8.9 6.3 4.1 11.3 4.4 22.6 9.3 8.4 3.5 13.7 9.7 10.5 18.2-2.6 7.1-11 14.4-22.7 18.1-11.1 3.6-19.8 16-38.2 14.9-3.9-.2-7-1-9.6-2.1-8-3.5-12.2-10.4-20-15-8.6-4.8-13.2-10.4-14.7-15.3-1.4-4.9 0-9 4.2-12.3zm3.3 334c-2.7 35.1-43.9 34.4-75.3 18-29.9-15.8-68.6-6.5-76.5-21.9-2.4-4.7-2.4-12.7 2.6-26.4l0-.2c2.4-7.6 .6-16-.6-23.9-1.2-7.8-1.8-15 .9-20 3.5-6.7 8.5-9.1 14.8-11.3 10.3-3.7 11.8-3.4 19.6-9.9 5.5-5.7 9.5-12.9 14.3-18 5.1-5.5 10-8.1 17.7-6.9 8.1 1.2 15.1 6.8 21.9 16l19.6 35.6c9.5 19.9 43.1 48.4 41 68.9zm-1.4-25.9c-4.1-6.6-9.6-13.6-14.4-19.6 7.1 0 14.2-2.2 16.7-8.9 2.3-6.2 0-14.9-7.4-24.9-13.5-18.2-38.3-32.5-38.3-32.5-13.5-8.4-21.1-18.7-24.6-29.9s-3-23.3-.3-35.2c5.2-22.9 18.6-45.2 27.2-59.2 2.3-1.7 .8 3.2-8.7 20.8-8.5 16.1-24.4 53.3-2.6 82.4 .6-20.7 5.5-41.8 13.8-61.5 12-27.4 37.3-74.9 39.3-112.7 1.1 .8 4.6 3.2 6.2 4.1 4.6 2.7 8.1 6.7 12.6 10.3 12.4 10 28.5 9.2 42.4 1.2 6.2-3.5 11.2-7.5 15.9-9 9.9-3.1 17.8-8.6 22.3-15 7.7 30.4 25.7 74.3 37.2 95.7 6.1 11.4 18.3 35.5 23.6 64.6 3.3-.1 7 .4 10.9 1.4 13.8-35.7-11.7-74.2-23.3-84.9-4.7-4.6-4.9-6.6-2.6-6.5 12.6 11.2 29.2 33.7 35.2 59 2.8 11.6 3.3 23.7 .4 35.7 16.4 6.8 35.9 17.9 30.7 34.8-2.2-.1-3.2 0-4Line truncated
+      <div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Mobile</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Mobile for iOS" title="ArchiveBox Mobile for iOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-android" href="https://android.archivebox.io/" aria-label="ArchiveBox Mobile for Android" title="ArchiveBox Mobile for Android"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!-- Font Awesome Free 5.0.13 by Fonticons, Inc. https://fontawesome.com License: https://creativecommons.org/licenses/by/4.0/ --><path d="M89.6 204.5v115.8c0 15.4-12.1 27.7-27.5 27.7-15.3 0-30.1-12.4-30.1-27.7V204.5c0-15.1 14.8-27.5 30.1-27.5 15.1 0 27.5 12.4 27.5 27.5zm10.8 157c0 16.4 13.2 29.6 29.6 29.6h19.9l.3 61.1c0 36.9 55.2 36.6 55.2 0v-61.1h37.2v61.1c0 36.7 55.5 36.8 55.5 0v-61.1h20.2c16.2 0 29.4-13.2 29.4-29.6V182.1H100.4v179.4zm248-189.1H99.3c0-42.8 25.6-80 63.6-99.4l-19.1-35.3c-2.8-4.9 4.3-8 6.7-3.8l19.4 35.6c34.9-15.5 75-14.7 108.3 0L297.5 34c2.5-4.3 9.5-1.1 6.7 3.8L285.1 73c37.7 19.4 63.3 56.6 63.3 99.4zm-170.7-55.5c0-5.7-4.6-10.5-10.5-10.5-5.7 0-10.2 4.8-10.2 10.5s4.6 10.5 10.2 10.5c5.9 0 10.5-4.8 10.5-10.5zm113.4 0c0-5.7-4.6-10.5-10.2-10.5-5.9 0-10.5 4.8-10.5 10.5s4.6 10.5 10.5 10.5c5.6 0 10.2-4.8 10.2-10.5zm94.8 60.1c-15.1 0-27.5 12.1-27.5 27.5v115.8c0 15.4 12.4 27.7 27.5 27.7 15.4 0 30.1-12.4 30.1-27.7V204.5c0-15.4-14.8-27.5-30.1-27.5z"/></svg></a></div></div>
+      <div class="abx-app-row"><a class="abx-app-name" href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><div class="abx-platforms"><a class="abx-platform abx-platform-chrome" href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj" aria-label="ArchiveBox Browser Extension for Chrome" title="ArchiveBox Browser Extension for Chrome"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc. --><path d="M0 256C0 209.4 12.47 165.6 34.27 127.1L144.1 318.3C166 357.5 207.9 384 256 384C270.3 384 283.1 381.7 296.8 377.4L220.5 509.6C95.9 492.3 0 385.3 0 256zM365.1 321.6C377.4 302.4 384 279.1 384 256C384 217.8 367.2 183.5 340.7 160H493.4C505.4 189.6 512 222.1 512 256C512 397.4 397.4 511.1 256 512L365.1 321.6zM477.8 128H256C193.1 128 142.3 172.1 130.5 230.7L54.19 98.47C101 38.53 174 0 256 0C350.8 0 433.5 51.48 477.8 128V128zM168 256C168 207.4 207.4 168 256 168C304.6 168 344 207.4 344 256C344 304.6 304.6 344 256 344C207.4 344 168 304.6 168 256z"/></svg></a><a class="abx-platform abx-platform-safari" href="https://extension.archivebox.io/" aria-label="ArchiveBox Browser Extension for Safari" title="ArchiveBox Browser Extension for Safari"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M274.7 274.7L237.3 237.3 166 346 274.7 274.7zM256 8a248 248 0 1 0 0 496 248 248 0 1 0 0-496zM411.8 182.8l14.8-6.1c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.8 1 2.6 1.7 1.3 1.6 1.7 2.6 .6 2 .6 3.1-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.3-2.6 1.7L418 197.6c-2 .8-4.2 .8-6.1 0s-3.5-2.4-4.3-4.3-.8-4.2 0-6.1 2.4-3.5 4.3-4.3zM314.4 94l6.1-14.8c.8-2 2.4-3.5 4.3-4.3s4.2-.8 6.1 0 3.5 2.4 4.3 4.3 .8 4.2 0 6.1l-6.1 14.8c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0c-1-.4-1.9-1-2.6-1.7s-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1zM256 60c2.1 0 4.2 .8 5.7 2.3S264 65.9 264 68l0 16c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7S253.9 60 256 60zM181 74.9c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.9 1 2.6 1.7 1.3 1.6 1.7 2.6L197.6 94c.4 1 .7 2 .7 3.1s-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.4-2.6 1.8-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.8-1.3-1.7-1.7-2.7l-6.1-14.8c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3zm-63.6 42.5c1.5-1.5 3.5-2.3 5.7-2.3s4.2 .8 5.7 2.3L140 128.7c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3l-11.3-11.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6l0 0zM60 256c0-2.1 .8-4.2 2.3-5.7S65.9 248 68 248l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3l-16 0c-2.1 0-4.2-.8-5.7-2.3S60 258.1 60 256zm40.2 73.2l-14.8 6.1c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1 1-1.9 1.7-2.6 1.6-1.3 2.6-1.7L94 314.4c2-.8 4.2-.8 6.1 0s3.5 2.4 4.3 4.3 .8 4.2 0 6.1-2.4 3.5-4.3 4.3zm4.3-136c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1 2.4-3.5 4.3-4.3 4.2-.8 6.1 0l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1l0 0zM197.6 418l-6.1 14.8c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1l6.1-14.8c.4-1 1-1.9 1.7-2.7s1.6-1.4 2.6-1.8 2-.6 3.1-.6 2.1 .2 3.1 .6 1.9 1 2.6 1.8 1.3 1.7 1.7 2.6 .6 2 .6 3.1-.2 2.1-.7 3.1zM264 444c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7s3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3 2.3 3.5 2.3 5.7l0 16zm67-6.9c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6L314.4 418c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3 4.2-.8 6.1 0 3.5 2.4 4.3 4.3l6.1 14.8c.8 2 .8 4.2 0 6.1s-2.4 3.5-4.3 4.3zm63.6-42.5c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3L372 383.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6 3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3l11.3 11.3c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6l0 0zM286.2 286.2L110.3 401.7 225.8 225.8 401.7 110.3 286.2 286.2zM437.1 331c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1c.4-1 1-1.9 1.7-2.6s1.6-1.3 2.6-1.7 2-.6 3.1-.6 2.1 .2 3.1 .6l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1zm6.9-67l-16 0c-2.1 0-4.2-.8-5.7-2.3s-2.3-3.5-2.3-5.7 .8-4.2 2.3-5.7 3.5-2.3 5.7-2.3l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3z"/></svg></a><a class="abx-platform abx-platform-firefox" href="https://addons.mozilla.org/firefox/addon/archivebox-exporter/" aria-label="ArchiveBox Browser Extension for Firefox" title="ArchiveBox Browser Extension for Firefox"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512Line truncated
+      <div class="abx-app-row abx-plugin-row"><a class="abx-app-name" href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><div class="abx-platforms abx-icon-stack"><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Google Drive storage" title="Google Drive storage"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path d="m38.7 419.3 22.6 39c4.7 8.2 11.4 14.7 19.4 19.4l80.6-139.6H0c0 9.1 2.3 18.2 7 26.4z" style="fill:#0066da"/><path d="M256 173.9 175.4 34.3c-7.9 4.7-14.7 11.1-19.4 19.4L7 311.7c-4.6 8-7 17.1-7 26.4h161.3z" style="fill:#00ac47"/><path d="M431.4 477.7c7.9-4.7 14.7-11.1 19.4-19.4l9.4-16.1 44.9-77.7c4.7-8.2 7-17.3 7-26.4H350.7l34.3 67.4z" style="fill:#ea4335"/><path d="m256 173.9 80.6-139.6c-7.9-4.7-17-7-26.4-7H201.8c-9.4 0-18.5 2.6-26.4 7z" style="fill:#00832d"/><path d="M350.7 338.1H161.3L80.6 477.7c7.9 4.7 17 7 26.4 7h298c9.4 0 18.5-2.6 26.4-7z" style="fill:#2684fc"/><path d="M430.5 182.7 356 53.7c-4.7-8.2-11.4-14.7-19.4-19.4L256 173.9l94.7 164.2h161c0-9.1-2.3-18.2-7-26.4z" style="fill:#ffba00"/></svg></a><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Amazon S3 storage" title="Amazon S3 storage"><svg aria-hidden="true" focusable="false" fill="#569A31" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M20.913 13.147l.12-.895c.947.576 1.258.922 1.354 1.071-.16.031-.562.046-1.474-.176zm-2.174 7.988a.547.547 0 0 0-.005.073c0 .084-.207.405-1.124.768a10.28 10.28 0 0 1-1.438.432c-1.405.325-3.128.504-4.853.504-4.612 0-7.412-1.184-7.412-1.704a.547.547 0 0 0-.005-.073L1.81 5.602c.135.078.28.154.432.227.042.02.086.038.128.057.134.062.272.122.417.18l.179.069c.154.058.314.114.478.168.043.013.084.029.13.043.207.065.423.127.646.187l.176.044c.175.044.353.087.534.127a23.414 23.414 0 0 0 .843.17l.121.023c.252.045.508.085.768.122.071.011.144.02.216.03.2.027.4.053.604.077l.24.027c.245.026.49.05.74.07l.081.009c.275.022.552.04.83.056l.233.012c.21.01.422.018.633.025a33.088 33.088 0 0 0 2.795-.026l.232-.011c.278-.016.555-.034.83-.056l.08-.008c.25-.02.497-.045.742-.072l.238-.026c.205-.024.408-.05.609-.077.07-.01.141-.019.211-.03.261-.037.519-.078.772-.122l.111-.02c.215-.04.427-.082.634-.125l.212-.047c.186-.041.368-.085.546-.13l.166-.042c.225-.06.444-.122.654-.189.04-.012.077-.026.115-.038a10.6 10.6 0 0 0 .493-.173c.058-.021.114-.044.17-.066.15-.06.293-.12.43-.185.038-.017.079-.034.116-.052.153-.073.3-.15.436-.228l-.976 7.245c-2.488-.78-5.805-2.292-7.311-3a1.09 1.09 0 0 0-1.088-1.085c-.6 0-1.088.489-1.088 1.088 0 .6.488 1.089 1.088 1.089.196 0 .378-.056.537-.148 1.72.812 5.144 2.367 7.715 3.15zm-7.42-20.047c5.677 0 9.676 1.759 9.75 2.736l-.014.113c-.01.033-.031.067-.048.101-.015.028-.026.057-.047.087-.024.033-.058.068-.09.102-.028.03-.051.06-.084.09-.038.035-.087.07-.133.105-.04.03-.074.06-.119.091-.053.036-.116.071-.177.107-.05.03-.095.06-.15.09-.068.036-.147.073-.222.11-.059.028-.114.057-.177.085-.084.038-.177.074-.268.111-.068.027-.13.054-.203.082-.097.036-.205.072-.31.107-.075.026-.148.053-.228.079-.111.035-.233.069-.35.103-.085.024-.165.05-.253.073-.124.034-.258.065-.389.098-.093.022-.181.046-.278.068-.139.032-.287.061-.433.091-.098.02-.191.041-.293.06-.155.03-.32.057-.482.084-.1.018-.198.036-.302.052-.166.026-.342.048-.515.072-.11.014-.213.03-.325.044-.181.023-.372.041-.56.06-.11.012-.218.025-.332.036-.188.016-.386.029-.58.043-.122.009-.24.02-.364.028-.207.012-.422.02-.635.028-.12.005-.234.012-.354.016a35.605 35.605 0 0 1-2.069 0c-.12-.004-.234-.011-.352-.016-.214-.008-.43-.016-.637-.028-.122-.008-.238-.02-.36-.027-.195-.015-.394-.028-.584-.044-.11-.01-.215-.024-.324-.035-.19-.02-.384-.038-.568-.06l-.315-.044c-.176-.024-.355-.046-.525-.073-.1-.015-.192-.033-.29-.05-.167-.028-.335-.055-.494-.086-.096-.018-.183-.038-.276-.056-.151-.032-.305-.062-.45-.095-.09-.02-.173-.043-.26-.064-.138-.034-.277-.067-.407-.102-.082-.022-.157-.046-.235-.069a11.75 11.75 0 0 1-.368-.108c-.075-.024-.141-.049-.213-.073-.11-.037-.223-.075-.325-.113-.067-.025-.125-.051-.188-.077-.096-.038-.195-.076-.282-.115-.06-.027-.11-.054-.166-.08-.08-.039-.162-.077-.233-.116-.052-.028-.094-.055-.142-.084-.063-.038-.13-.075-.185-.113-.043-.029-.075-.058-.113-.086-.048-.037-.098-.073-.139-.11-.032-.029-.054-.057-.08-.087-.033-.035-.069-.07-.093-.104-.02-.03-.031-.058-.046-.086-.018-.035-.039-.068-.049-.102l-.015-.113c.076-.977 4.074-2.736 9.748-2.736zm12.182 12.124c-.118-.628-.84-1.291-2.31-2.128l.963-7.16a.531.531 0 0 0 .005-.073C22.16 1.581 16.447 0 11.32 0 6.194 0 .482 1.581.482 3.851a.58.58 0 0 0 .005.072L2.819 21.25c.071 2.002 5.236 2.75 8.5 2.75 1.805 0 3.615-.188 5.098-.531.598-.138 1.133-.3 1.592-.48 1.18-.467 1.789-1.053 1.813-1.739l.945-7.018c.557.131 1.016.197 1.389.197.54 0 .902-.137 1.134-.413aLine truncated
+      <div class="abx-app-row abx-tools-row"><a class="abx-app-name" href="https://abx-dl.archivebox.io/">Tools</a><div class="abx-platforms"><a class="abx-tool" href="https://abx-dl.archivebox.io/"><span>One-shot <code>abx-dl</code> CLI</span></a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/tree/dev/archivebox/mcp">MCP</a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-api-usage">API</a><a class="abx-tool" href="https://web.archivebox.io">Demo</a></div></div>
+    </div></details>
+    __SITE_NAV__
+  </nav>
+</header>

.github/pages/base/marquee.css

@@ -0,0 +1,11 @@
+/* Identical standalone gallery strip across ArchiveBox product sites. */
+/* The root query container measures the usable width, excluding scrollbars. */
+html { container-type: inline-size; }
+.abx-marquee { min-width: 0; width: 100cqw; max-width: none; margin: 24px calc(50% - 50cqw) 40px; padding: 0; overflow: hidden; }
+.abx-marquee .abx-marquee-viewport { width: 100%; overflow-x: auto; scroll-behavior: auto; overscroll-behavior-x: contain; scrollbar-width: thin; padding: 16px 24px 28px; box-sizing: border-box; }
+.abx-marquee .abx-marquee-track { display: flex; align-items: flex-start; gap: 24px; width: max-content; margin-inline: auto; }
+.abx-marquee .abx-marquee-card { display: block; flex: none; width: min-content; overflow: hidden; border-radius: 10px; background: #fff; box-shadow: 0 0 0 1px rgb(0 0 0 / 12%), 0 2px 4px rgb(0 0 0 / 10%), 0 8px 16px rgb(0 0 0 / 16%); text-decoration: none; color: #533344; }
+.abx-marquee .abx-marquee-card img { display: block; height: clamp(240px, 30vw, 400px); width: auto; max-width: none; max-height: none; object-fit: contain; box-shadow: none; border: 0; border-radius: 0; background: transparent; padding: 0; margin: 0; }
+.abx-marquee :focus-visible { outline: 2px solid #9b2854; outline-offset: 3px; }
+.abx-marquee .abx-marquee-viewport:focus-visible { outline-offset: -3px; }
+@media (max-width: 600px) { .abx-marquee .abx-marquee-track { gap: 16px; } .abx-marquee .abx-marquee-card img { height: 240px; } }

.github/pages/base/marquee.js

@@ -0,0 +1,44 @@
+/* Scroll real gallery links once; keep them usable without JavaScript. */
+(() => {
+  document.querySelectorAll('.abx-marquee').forEach(gallery => {
+    const viewport = gallery.querySelector('.abx-marquee-viewport');
+    const reducedMotion = window.matchMedia('(prefers-reduced-motion: reduce)');
+    let paused = reducedMotion.matches;
+    let hovering = false;
+    let focused = false;
+    let visible = false;
+    let frame;
+    let previous;
+    let offset = viewport.scrollLeft;
+    function tick(time) {
+      const distance = viewport.scrollWidth - viewport.clientWidth;
+      if (distance <= 0) return;
+      if (previous !== undefined) {
+        offset = Math.min(offset + Math.min(time - previous, 50) * 0.035, distance);
+        viewport.scrollTo({ left: offset, behavior: 'instant' });
+      }
+      if (offset >= distance - 1) { paused = true; return; }
+      previous = time;
+      frame = window.requestAnimationFrame(tick);
+    }
+    function sync() {
+      window.cancelAnimationFrame(frame);
+      previous = undefined;
+      offset = viewport.scrollLeft;
+      if (!paused && !hovering && !focused && visible && !document.hidden) frame = window.requestAnimationFrame(tick);
+    }
+    viewport.addEventListener('pointerenter', event => { if (event.pointerType === 'mouse') { hovering = true; sync(); } });
+    viewport.addEventListener('pointerleave', () => { hovering = false; sync(); });
+    viewport.addEventListener('focusin', () => { focused = true; sync(); });
+    viewport.addEventListener('focusout', event => { focused = viewport.contains(event.relatedTarget); sync(); });
+    function pauseForInteraction() { paused = true; sync(); }
+    viewport.addEventListener('pointerdown', pauseForInteraction);
+    viewport.addEventListener('wheel', pauseForInteraction, { passive: true });
+    viewport.addEventListener('keydown', pauseForInteraction);
+    reducedMotion.addEventListener('change', () => { paused = reducedMotion.matches; sync(); });
+    document.addEventListener('visibilitychange', sync);
+    new window.IntersectionObserver(([entry]) => { visible = entry.isIntersecting; sync(); }).observe(gallery);
+    new window.ResizeObserver(sync).observe(viewport);
+    viewport.querySelectorAll('img').forEach(image => image.addEventListener('load', sync));
+  });
+})();

.github/pages/marquee.py

@@ -0,0 +1,96 @@
+"""Render homepage screenshot strips from the site's existing gallery HTML."""
+
+import re
+from html import escape
+from html.parser import HTMLParser
+from urllib.parse import urljoin
+
+
+class Gallery(HTMLParser):
+    """Only gallery figures are captures; navigation and decorative images are not."""
+
+    def __init__(self, base):
+        super().__init__()
+        self.base = base
+        self.stack = []
+        self.captures = []
+
+    def handle_starttag(self, tag, attrs):
+        attrs = dict(attrs)
+        if tag == "img" and any(name == "figure" for name, _ in self.stack):
+            context = [item for _, item in self.stack]
+            profile = next(
+                (
+                    item.get("data-profile") or item.get("data-viewport")
+                    for item in reversed(context)
+                    if item.get("data-profile") or item.get("data-viewport")
+                ),
+                None,
+            )
+            if profile not in (None, "desktop"):
+                return
+            source = attrs.get("src", "")
+            if not source or not attrs.get("alt"):
+                return
+            anchor = next((item["id"] for name, item in reversed(self.stack) if name in ("article", "figure") and item.get("id")), "")
+            server = any(item.get("data-platform") == "server" for item in context)
+            self.captures.append(
+                {
+                    "src": urljoin(self.base, source),
+                    "href": self.base + "#" + anchor if anchor else urljoin(self.base, source),
+                    "title": attrs["alt"],
+                    "width": attrs.get("width", ""),
+                    "height": attrs.get("height", ""),
+                    "product": "server" if server else "client",
+                },
+            )
+        if tag not in {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}:
+            self.stack.append((tag, attrs))
+
+    def handle_endtag(self, tag):
+        for index in range(len(self.stack) - 1, -1, -1):
+            if self.stack[index][0] == tag:
+                del self.stack[index:]
+                break
+
+
+def render(output, baseurl="", featured=()):
+    homepage = output / "index.html"
+    text = homepage.read_text()
+    slots = re.findall(r"<!-- ARCHIVEBOX:(MARQUEE(?:-CLIENT|-SERVER)?) -->", text)
+    if not slots:
+        return False
+    base = "/" + baseurl.strip("/") + "/" if baseurl.strip("/") else "/"
+    gallery = Gallery(base + "screenshots/")
+    gallery.feed((output / "screenshots/index.html").read_text())
+    for slot in slots:
+        product = slot.removeprefix("MARQUEE-").lower() if "-" in slot else None
+        captures = [capture for capture in gallery.captures if product is None or capture["product"] == product]
+        priority = {title: index for index, title in enumerate(featured)}
+        captures.sort(key=lambda capture: priority.get(capture["title"], len(priority)))
+        cards = []
+        for capture in captures:
+            dimensions = "".join(f' {key}="{escape(capture[key])}"' for key in ("width", "height") if capture[key])
+            cards.append(
+                f'<a class="abx-marquee-card" href="{escape(capture["href"])}"><img src="{escape(capture["src"])}" alt="{escape(capture["title"])}"{dimensions} loading="lazy" decoding="async"></a>',
+            )
+        viewport_id = slot.lower()
+        content = (
+            f'<section class="abx-marquee" aria-label="App screenshots"><div class="abx-marquee-viewport" id="{viewport_id}" tabindex="0" aria-label="Screenshots. Scroll sideways to explore."><div class="abx-marquee-track">{"".join(cards)}</div></div></section>'
+            if cards
+            else ""
+        )
+        start, end = f"<!-- ARCHIVEBOX:{slot} -->", f"<!-- /ARCHIVEBOX:{slot} -->"
+        replacement = start + "\n" + content + "\n" + end
+        if end in text:
+            text = re.sub(
+                re.escape(start) + r".*?" + re.escape(end),
+                lambda _, replacement=replacement: replacement,
+                text,
+                count=1,
+                flags=re.DOTALL,
+            )
+        else:
+            text = text.replace(start, replacement, 1)
+    homepage.write_text(text)
+    return True

.github/pages/nav.html

@@ -0,0 +1,4 @@
+<!-- Repository-specific navigation and primary CTA. -->
+<a href="__SITE_BASE__#use-cases">Use cases</a><a href="__SITE_BASE__#configuration">Configuration</a><a href="__SITE_BASE__#layout-security">Layout &amp; security</a><a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
+    <a href="https://github.com/ArchiveBox/ArchiveBox">GitHub ↗</a>
+    <a class="abx-cta" href="__SITE_BASE__#install">Get started</a>

.github/pages/site.json

@@ -0,0 +1,16 @@
+{
+  "name": "Web Archive",
+  "url": "https://archivebox.io/",
+  "pages": [
+    "index.html",
+    "screenshots/index.html"
+  ],
+  "marquee_featured": [
+    "Snapshots grid — desktop",
+    "Snapshot View (header collapsed) — desktop",
+    "AI agent — desktop",
+    "Archive results — desktop",
+    "Add URLs — desktop",
+    "Snapshots table — desktop"
+  ]
+}

.github/pages/site.py

@@ -0,0 +1,215 @@
+"""Assemble and verify this repository's static Pages site (standard library only)."""
+
+import argparse
+import json
+import re
+import shutil
+import subprocess
+from datetime import UTC, datetime
+from html import escape, unescape
+from html.parser import HTMLParser
+from pathlib import Path
+from urllib.parse import unquote, urlsplit
+
+from marquee import render as render_marquee
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parents[1]
+
+
+def replace_block(text, name, content):
+    start, end = f"<!-- ARCHIVEBOX:{name} -->", f"<!-- /ARCHIVEBOX:{name} -->"
+    pattern = re.escape(start) + r".*?" + re.escape(end)
+    if re.search(pattern, text, re.DOTALL):
+        return re.sub(
+            pattern,
+            lambda _: f"{start}\n{content}\n{end}",
+            text,
+            count=1,
+            flags=re.DOTALL,
+        )
+    if text.count(start) != 1:
+        raise ValueError(f"Expected one {name} placeholder")
+    return text.replace(start, f"{start}\n{content}\n{end}", 1)
+
+
+def render_metadata(text):
+    """Keep product copy in native templates; render one consistent SEO block."""
+    page = Page()
+    page.feed(text)
+    title = unescape(re.search(r"<title>(.*?)</title>", text, re.DOTALL)[1])
+    meta = page.metadata
+    description = meta.get("description", meta.get("og:description", ""))
+    image = meta.get("og:image", meta.get("twitter:image", ""))
+    if not description or not image or not page.canonical:
+        raise ValueError("Each page needs a description, social image and canonical")
+    values = {
+        "description": description,
+        "robots": "index,follow,max-image-preview:large",
+        "theme-color": "#9b2854",
+        "og:type": "website",
+        "og:site_name": meta.get("og:site_name", "ArchiveBox"),
+        "og:locale": meta.get("og:locale", "en_US"),
+        "og:url": page.canonical,
+        "og:title": title,
+        "og:description": description,
+        "og:image": image,
+        "og:image:type": meta.get("og:image:type", "image/png"),
+        "og:image:width": meta.get("og:image:width", "1200"),
+        "og:image:height": meta.get("og:image:height", "630"),
+        "og:image:alt": meta.get("og:image:alt", title),
+        "twitter:card": "summary_large_image",
+        "twitter:site": "@ArchiveBoxApp",
+        "twitter:title": title,
+        "twitter:description": description,
+        "twitter:image": image,
+        "twitter:image:alt": meta.get("og:image:alt", title),
+    }
+
+    def strip_managed_tag(match):
+        tag = Page()
+        tag.feed(match[0])
+        if tag.canonicals or any(key in values for key in tag.metadata):
+            return ""
+        # System fonts are shared across sites, without a network font dependency.
+        if "fonts.googleapis.com" in match[0] or "fonts.gstatic.com" in match[0]:
+            return ""
+        return match[0]
+
+    text = re.sub(r"<(?:meta|link)\b[^>]*>", strip_managed_tag, text)
+    text = re.sub(r"<title>.*?</title>", "", text, flags=re.DOTALL)
+    tags = [
+        f"<title>{escape(title)}</title>",
+        f'<link rel="canonical" href="{escape(page.canonical, quote=True)}">',
+    ]
+    for key, value in values.items():
+        attr = "property" if key.startswith("og:") else "name"
+        tags.append(f'<meta {attr}="{key}" content="{escape(value, quote=True)}">')
+    if "<!-- ARCHIVEBOX:METADATA -->" not in text:
+        text = text.replace("</head>", "<!-- ARCHIVEBOX:METADATA -->\n</head>", 1)
+    return replace_block(text, "METADATA", "\n".join(tags))
+
+
+def render(output, source=None, baseurl=""):
+    config = json.loads((HERE / "site.json").read_text())
+    output = output.resolve()
+    if source:
+        source = source.resolve()
+        if output == source or output in source.parents or source in output.parents:
+            raise ValueError("Build output must be separate from the source directory")
+        if output == ROOT or output in ROOT.parents:
+            raise ValueError("Build output must not replace a repository")
+        if output.exists():
+            shutil.rmtree(output)
+        shutil.copytree(source, output)
+    revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
+    assets = output / "site-base"
+    assets.mkdir(parents=True, exist_ok=True)
+    has_marquee = render_marquee(output, baseurl, config.get("marquee_featured", []))
+    common_assets = ["chrome.css", "chrome.js"] + (["marquee.css", "marquee.js"] if has_marquee else [])
+    for name in common_assets:
+        shutil.copyfile(HERE / "base" / name, assets / name)
+    for name in config["pages"]:
+        page = output / name
+        text = render_metadata(page.read_text())
+        base = "../" * (len(Path(name).parts) - 1) or "./"
+        header = (HERE / "base/header.html").read_text().replace("__SITE_NAME__", escape(config["name"]))
+        header = header.replace("__SITE_NAV__", (HERE / "nav.html").read_text().strip())
+        text = replace_block(text, "HEADER", header)
+        for kind in ["FOOTER-START", "FOOTER-END"]:
+            text = replace_block(text, kind, (HERE / "base" / f"{kind.lower()}.html").read_text().strip())
+        text = text.replace("__SITE_BASE__", base).replace("__SITE_REVISION__", revision)
+        resources = f'<link rel="stylesheet" href="{base}site-base/chrome.css?v={revision}">\n<script src="{base}site-base/chrome.js?v={revision}" defer></script>'
+        if has_marquee and name == "index.html":
+            resources += f'\n<link rel="stylesheet" href="{base}site-base/marquee.css?v={revision}">\n<script src="{base}site-base/marquee.js?v={revision}" defer></script>'
+        if "<!-- ARCHIVEBOX:ASSETS -->" not in text:
+            text = text.replace("</head>", "<!-- ARCHIVEBOX:ASSETS -->\n</head>", 1)
+        text = replace_block(text, "ASSETS", resources)
+        page.write_text(text)
+    # Site build provenance is independent of all original screenshot manifests.
+    provenance_path = output / "build.json"
+    provenance = json.loads(provenance_path.read_text()) if provenance_path.exists() else {}
+    provenance.update(
+        revision=revision,
+        generatedAt=datetime.now(UTC).isoformat(),
+        site=config["url"],
+    )
+    provenance_path.write_text(json.dumps(provenance, indent=2) + "\n")
+    (output / ".nojekyll").touch()
+    verify(output, baseurl)
+    print(f"Assembled {len(config['pages'])} pages at {output} from {revision}")
+
+
+class Page(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.ids = set()
+        self.metadata = {}
+        self.canonical = ""
+        self.resources = []
+        self.headers = self.footers = self.canonicals = 0
+
+    def handle_starttag(self, tag, attrs):
+        attrs = dict(attrs)
+        if tag == "meta":
+            for key in ["name", "property"]:
+                if attrs.get(key):
+                    self.metadata[attrs[key]] = attrs.get("content", "")
+        if tag == "link" and attrs.get("rel") == "canonical":
+            self.canonical = attrs["href"]
+        if attrs.get("id"):
+            self.ids.add(attrs["id"])
+        classes = attrs.get("class", "").split()
+        self.headers += "abx-header" in classes
+        self.footers += "abx-footer" in classes
+        self.canonicals += tag == "link" and attrs.get("rel") == "canonical"
+        if tag in ["img", "script"] and attrs.get("src"):
+            self.resources.append(attrs["src"])
+        if tag == "link" and attrs.get("rel") in [
+            "stylesheet",
+            "icon",
+            "apple-touch-icon",
+        ]:
+            self.resources.append(attrs["href"])
+
+
+def verify(output, baseurl=""):
+    config = json.loads((HERE / "site.json").read_text())
+    output = output.resolve()
+    prefix = "/" + baseurl.strip("/") if baseurl.strip("/") else ""
+    for name in config["pages"]:
+        path = output / name
+        text = path.read_text()
+        page = Page()
+        page.feed(text)
+        if (page.headers, page.footers, page.canonicals) != (1, 1, 1):
+            raise ValueError(f"{name}: expected exactly one shared header, footer and canonical")
+        if "__SITE_" in text:
+            raise ValueError(f"{name}: unresolved site placeholder")
+        for resource in page.resources:
+            url = urlsplit(resource)
+            if url.scheme or url.netloc or not url.path:
+                continue
+            resource_path = unquote(url.path)
+            if prefix and resource_path.startswith(prefix + "/"):
+                resource_path = resource_path[len(prefix) :]
+            local = (output / resource_path.lstrip("/") if resource_path.startswith("/") else path.parent / resource_path).resolve()
+            if not local.is_relative_to(output) or not local.is_file():
+                raise ValueError(f"{name}: missing local resource {resource}")
+        for asset in ["chrome.css", "chrome.js"]:
+            if (output / "site-base" / asset).read_bytes() != (HERE / "base" / asset).read_bytes():
+                raise ValueError(f"{name}: stale common asset {asset}")
+    print(f"Verified shared layout and local resources for {len(config['pages'])} pages")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("command", choices=["render", "verify"])
+    parser.add_argument("output", type=Path)
+    parser.add_argument("--source", type=Path)
+    parser.add_argument("--baseurl", default="")
+    args = parser.parse_args()
+    if args.command == "render":
+        render(args.output, args.source, args.baseurl)
+    else:
+        verify(args.output, args.baseurl)

.github/pages/verify.py

@@ -0,0 +1,255 @@
+"""Check the built site in real Chromium at desktop and phone widths."""
+
+import argparse
+import json
+import re
+from functools import partial
+from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+from html.parser import HTMLParser
+from pathlib import Path
+from threading import Thread
+from urllib.parse import unquote, urljoin, urlsplit
+
+from playwright.sync_api import expect, sync_playwright
+
+HERE = Path(__file__).resolve().parent
+
+
+class Handler(SimpleHTTPRequestHandler):
+    def handle(self):
+        # Browser navigation can cancel an in-flight lazy image response.
+        try:
+            super().handle()
+        except (BrokenPipeError, ConnectionResetError):
+            pass
+
+    def log_message(self, *args):
+        pass
+
+
+class ScreenshotReferences(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.references = []
+        self.ids = set()
+
+    def handle_starttag(self, tag, attributes):
+        attrs = dict(attributes)
+        if attrs.get("id"):
+            self.ids.add(attrs["id"])
+        for attribute in ("src", "href"):
+            if attrs.get(attribute):
+                self.references.append(attrs[attribute])
+
+
+def verify_screenshot_references(output, canonical):
+    """Check gallery embeds on every static page, including optional translations."""
+    pages = {}
+    for path in output.rglob("*.html"):
+        page = ScreenshotReferences()
+        page.feed(path.read_text())
+        pages[path.resolve()] = page
+    for path, page in pages.items():
+        page_url = urljoin(canonical, path.relative_to(output.resolve()).as_posix())
+        for reference in page.references:
+            url = urlsplit(urljoin(page_url, reference))
+            if url.netloc != urlsplit(canonical).netloc or "/screenshots/" not in url.path:
+                continue
+            target = output / unquote(url.path).lstrip("/")
+            if target.is_dir():
+                target /= "index.html"
+            target = target.resolve()
+            assert target.is_relative_to(output.resolve()) and target.is_file(), f"{path}: missing screenshot target {reference}"
+            assert not re.match(r"^\d+-", target.name), f"{path}: screenshot filename depends on capture order: {reference}"
+            if url.fragment and target in pages:
+                assert unquote(url.fragment) in pages[target].ids, f"{path}: missing screenshot anchor {reference}"
+
+
+def verify(output, evidence):
+    config = json.loads((HERE / "site.json").read_text())
+    verify_screenshot_references(output, config["url"])
+    evidence.mkdir(parents=True, exist_ok=True)
+    server = ThreadingHTTPServer(("127.0.0.1", 0), partial(Handler, directory=str(output.resolve())))
+    thread = Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    origin = f"http://127.0.0.1:{server.server_port}"
+    try:
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(ignore_default_args=["--hide-scrollbars"])
+            for name in config["pages"]:
+                route = name.removesuffix("index.html")
+                page = browser.new_page(reduced_motion="reduce")
+                errors = []
+                missing = []
+                page.on("pageerror", lambda error, errors=errors: errors.append(str(error)))
+                page.on(
+                    "response",
+                    lambda response, missing=missing: (
+                        missing.append(response.url) if response.url.startswith(origin) and response.status >= 400 else None
+                    ),
+                )
+                for width in [1716, 2560, 390]:
+                    page.set_viewport_size({"width": width, "height": 900})
+                    page.goto(f"{origin}/{route}", wait_until="domcontentloaded")
+                    header = page.locator(".abx-header")
+                    expect(header).to_have_count(1)
+                    expect(header).to_be_visible()
+                    expect(header).to_be_in_viewport()
+                    brand = header.locator(".abx-brand")
+                    expect(brand).to_be_visible()
+                    expect(brand).to_contain_text("ArchiveBox")
+                    expect(brand.locator(".abx-logo")).to_be_visible()
+                    expect(brand.locator(".abx-logo")).to_be_in_viewport()
+                    expect(header).to_have_css("display", "flex")
+                    expect(page.locator(".abx-footer")).to_have_count(1)
+                    expect(page.locator(".abx-footer-column")).to_have_count(3)
+                    expect(header.locator(".abx-cta")).to_be_visible()
+                    menu = header.locator(".abx-apps")
+                    menu.locator("summary").focus()
+                    page.keyboard.press("Enter")
+                    expect(menu).to_have_attribute("open", "")
+                    expect(menu.locator("a").first).to_be_visible()
+                    page.keyboard.press("Escape")
+                    expect(menu).not_to_have_attribute("open", "")
+                    menu.locator("summary").click()
+                    box = header.bounding_box()
+                    assert box
+                    page.mouse.click(box["x"] + 2, box["y"] + 5)
+                    expect(menu).not_to_have_attribute("open", "")
+                    assert page.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth"), (
+                        f"{name}: overflow at {width}"
+                    )
+                    if not route and "screenshots/index.html" in config["pages"]:
+                        strips = page.locator(".abx-marquee")
+                        assert strips.count() > 0, "Homepage must use its screenshot gallery"
+                        for strip in strips.all():
+                            strip.scroll_into_view_if_needed()
+                            bounds = strip.bounding_box()
+                            usable_width = page.evaluate("document.documentElement.clientWidth")
+                            assert bounds and abs(bounds["x"]) < 1
+                            assert abs(bounds["width"] - usable_width) < 1
+                            assert not strip.inner_text().strip(), "Screenshot strips should have no visible captions or links"
+                            cards = strip.locator(".abx-marquee-card")
+                            assert cards.count() > 0, "Screenshot strip is empty"
+                            expect(strip.locator("button")).to_have_count(0)
+                            expect(strip.locator("a:not(.abx-marquee-card)")).to_have_count(0)
+                            viewport = strip.locator(".abx-marquee-viewport")
+                            overflowing = viewport.evaluate(
+                                "node => node.scrollWidth > node.clientWidth",
+                            )
+                            if overflowing:
+                                page.mouse.move(0, 0)
+                                page.emulate_media(reduced_motion="no-preference")
+                                initial = viewport.evaluate("node => node.scrollLeft")
+                                page.wait_for_function(
+                                    "([node, initial]) => node.scrollLeft > initial",
+                                    arg=[viewport.element_handle(), initial],
+                                )
+                                viewport.hover()
+                                stopped = viewport.evaluate("node => node.scrollLeft")
+                                viewport.evaluate(
+                                    "node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
+                                )
+                                assert viewport.evaluate("node => node.scrollLeft") == stopped
+                                page.emulate_media(reduced_motion="reduce")
+                                page.mouse.move(0, 0)
+                                stopped = viewport.evaluate("node => node.scrollLeft")
+                                viewport.evaluate(
+                                    "node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
+                                )
+                                assert viewport.evaluate("node => node.scrollLeft") == stopped
+                                viewport.focus()
+                                page.keyboard.press("ArrowRight")
+                                assert viewport.evaluate("node => node.scrollWidth > node.clientWidth")
+                            else:
+                                centered = viewport.evaluate(
+                                    """node => {
+                                        const view = node.getBoundingClientRect();
+                                        const track = node.querySelector('.abx-marquee-track').getBoundingClientRect();
+                                        return Math.abs((track.left - view.left) - (view.right - track.right)) < 2;
+                                    }""",
+                                )
+                                assert centered, "Short screenshot strips must be centered"
+                            for card in cards.all():
+                                href = card.get_attribute("href")
+                                assert href and "/screenshots/" in href
+                                response = page.request.get(urljoin(page.url, href))
+                                assert response.ok, f"Broken screenshot link: {href}"
+                                anchor = urlsplit(href).fragment
+                                if anchor:
+                                    assert f'id="{anchor}"' in response.text(), f"Missing screenshot anchor: {href}"
+                    # Decode every local content image using the real browser loader,
+                    # including images in offscreen carousel tracks and lazy galleries.
+                    images = page.locator("main img").evaluate_all("nodes => [...new Set(nodes.map(image => image.src))]")
+                    local_images = [url for url in images if url.startswith(origin + "/")]
+                    page.evaluate(
+                        """async urls => {
+                        for (const url of urls) {
+                            const image = new Image();
+                            image.src = url;
+                            await image.decode();
+                            if (!image.naturalWidth) throw new Error(`Broken image: ${url}`);
+                        }
+                    }""",
+                        local_images,
+                    )
+                    page.evaluate("scrollTo({top: 0, behavior: 'instant'})")
+                    page.evaluate(
+                        "() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
+                    )
+                    page.screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}{width}.png"))
+                    page.locator(".abx-footer").screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}footer-{width}.png"))
+                    for link in header.locator(".abx-nav > a").all():
+                        url = urlsplit(link.get_attribute("href") or "")
+                        if url.scheme or url.netloc:
+                            continue
+                        response = page.request.get(urljoin(page.url, url.geturl()))
+                        assert response.ok, f"Broken navigation: {url.geturl()}"
+                        if url.fragment:
+                            body = response.text()
+                            assert f'id="{url.fragment}"' in body or f"id='{url.fragment}'" in body, (
+                                f"Missing navigation anchor: {url.geturl()}"
+                            )
+                    if route == "" and page.locator("#resources").count():
+                        page.goto(f"{origin}/#resources")
+                        expect(page.locator("#resources")).to_be_visible()
+                assert not errors, errors
+                assert not missing, missing
+                page.close()
+                context = browser.new_context(java_script_enabled=False, viewport={"width": 390, "height": 900})
+                plain = context.new_page()
+                plain.goto(f"{origin}/{route}", wait_until="domcontentloaded")
+                expect(plain.locator(".abx-header")).to_be_visible()
+                expect(plain.locator(".abx-header .abx-brand")).to_be_in_viewport()
+                expect(plain.locator(".abx-header .abx-logo")).to_be_visible()
+                plain.locator(".abx-apps summary").click()
+                expect(plain.locator(".abx-app-links a").first).to_be_visible()
+                expect(plain.locator(".abx-footer-column a").first).to_be_visible()
+                if not route and "screenshots/index.html" in config["pages"]:
+                    assert plain.locator(".abx-marquee-card").count() > 0
+                    for strip in plain.locator(".abx-marquee").all():
+                        bounds = strip.bounding_box()
+                        usable_width = plain.evaluate("document.documentElement.clientWidth")
+                        assert bounds and abs(bounds["x"]) < 1
+                        assert abs(bounds["width"] - usable_width) < 1
+                    assert plain.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth")
+                    expect(plain.locator(".abx-marquee button")).to_have_count(0)
+                    expect(plain.locator(".abx-marquee-card").first).to_be_visible()
+                context.close()
+                print(
+                    f"PASS {name}: desktop/mobile, keyboard, no-JS, images and resources",
+                    flush=True,
+                )
+            browser.close()
+    finally:
+        server.shutdown()
+        server.server_close()
+        thread.join()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("output", type=Path)
+    parser.add_argument("--evidence", type=Path, default=Path("test-results/site"))
+    args = parser.parse_args()
+    verify(args.output, args.evidence)

.github/workflows/ci.yml

@@ -2,6 +2,7 @@ name: CI
 
 on:
   pull_request:
+    branches: [dev, main]
   push:
     branches: [dev, main]
   workflow_dispatch:
@@ -51,7 +52,7 @@ jobs:
     name: Tested Docker images
     uses: ./.github/workflows/docker.yml
     with:
-      push_digests: false
+      push_digests: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
     secrets: inherit
 
   required:

.github/workflows/deploy-publicsite.yml

@@ -1,4 +1,5 @@
-name: Deploy Publicsite to GitHub Pages
+name: Publish website
+
 on:
   push:
     branches: [dev]
@@ -8,37 +9,73 @@ on:
     workflows: ['Capture publicsite screenshots']
     types: [completed]
     branches: [dev]
+
 permissions:
   contents: read
   actions: read
+
+env:
+  # Repository-specific output; all publication logic below is a local copy.
+  SITE_OUTPUT: _site
+  DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+
+# One production queue covers restore, build and deploy. PRs have separate queues.
 concurrency:
-  # Failed captures must not displace a pending push deployment.
-  group: fast-pages-${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'success' && github.run_id || github.ref }}
+  group: pages-${{ github.event_name == 'pull_request' && github.ref || 'production' }}
   cancel-in-progress: false
+
 jobs:
   build:
-    if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 20
+    outputs:
+      revision: ${{ steps.source.outputs.revision }}
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: ${{ github.event_name == 'pull_request' && github.ref || 'dev' }}
+          ref: ${{ github.event_name == 'pull_request' && github.ref || env.DEFAULT_BRANCH }}
+          persist-credentials: false
+      - name: Record website source revision
+        id: source
+        run: echo "revision=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      # BEGIN REPOSITORY-SPECIFIC BUILD
       - uses: astral-sh/setup-uv@v7
       - name: Restore latest complete screenshots
-        run: uv run --no-project python bin/restore_site_screenshots.py archivebox publicsite/screenshots
+        run: uv run --no-project python bin/restore_site_screenshots.py publicsite/screenshots
         env:
           GH_TOKEN: ${{ github.token }}
       - name: Render current gallery layout without recapturing images
         run: uv run --no-project python bin/generate_ui_screenshot_gallery.py refresh publicsite/screenshots/index.html
-      - uses: actions/upload-pages-artifact@v3
+      - name: Assemble website with the local base components
+        run: uv run --no-project python .github/pages/site.py render _site --source publicsite
+      # END REPOSITORY-SPECIFIC BUILD
+
+      - name: Verify website in Chromium
+        run: |
+          uv run --no-config --no-project --with playwright==1.63.0 playwright install --with-deps chromium
+          uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py "$SITE_OUTPUT" --evidence "$RUNNER_TEMP/site-evidence"
+      - name: Save website verification evidence
+        if: always()
+        uses: actions/upload-artifact@v4
         with:
-          path: publicsite
+          name: site-verification
+          path: ${{ runner.temp }}/site-evidence
+          if-no-files-found: warn
+          retention-days: 14
+
+      - name: Upload website
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ${{ env.SITE_OUTPUT }}
+
   deploy:
     if: github.event_name != 'pull_request'
     needs: build
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     permissions:
+      contents: read
       pages: write
       id-token: write
     environment:

.github/workflows/docker.yml

@@ -13,6 +13,22 @@ on:
         required: false
         default: true
         type: boolean
+  workflow_dispatch:
+    inputs:
+      source_ref:
+        description: Exact 40-character commit SHA to build
+        required: true
+        type: string
+      push_digests:
+        description: Push validated per-platform images by digest for the release workflow
+        required: false
+        default: true
+        type: boolean
+      full_tests:
+        description: Run storage, CLI, and size validation in addition to release smoke checks
+        required: false
+        default: true
+        type: boolean
 
 env:
   DOCKERHUB_IMAGE: archivebox/archivebox
@@ -42,6 +58,18 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          ref: ${{ inputs.source_ref || github.sha }}
+
+      - name: Verify exact source commit
+        if: github.event_name == 'workflow_dispatch'
+        shell: bash
+        env:
+          SOURCE_REF: ${{ inputs.source_ref }}
+        run: |
+          set -Eeuo pipefail
+          [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$(git rev-parse HEAD)" == "$SOURCE_REF" ]]
 
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
@@ -183,7 +211,7 @@ jobs:
           {
             echo 'labels<<EOF'
             echo "org.opencontainers.image.version=${VERSION}"
-            echo "org.opencontainers.image.revision=${GITHUB_SHA}"
+            echo "org.opencontainers.image.revision=${{ inputs.source_ref || github.sha }}"
             echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
             echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
             echo 'EOF'
@@ -203,7 +231,7 @@ jobs:
           labels: ${{ steps.docker_meta.outputs.labels }}
           build-args: |
             ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
+            ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
           pull: true
           platforms: ${{ matrix.platform }}
           # Keep the released image as the one cache authority. A second GHA
@@ -232,7 +260,7 @@ jobs:
           labels: ${{ steps.docker_meta.outputs.labels }}
           build-args: |
             ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
+            ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
           pull: true
           platforms: ${{ matrix.platform }}
           # Test the same layer lineage users receive. Reusing :dev avoids a
@@ -246,13 +274,34 @@ jobs:
         if: inputs.push_digests
         run: echo ${{ steps.docker_build.outputs.digest }}
 
+      - name: Verify OpenCode install with a read-only browser profile
+        shell: bash
+        env:
+          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
+        run: |
+          set -Eeuo pipefail
+          PROFILE_DIR="$(mktemp -d)"
+          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' \
+            --volume "$PROFILE_DIR:/home/archivebox/.config/chromium:ro" \
+            "$IMAGE" bash -c '
+              set -Eeuo pipefail
+              test -w "$XDG_CONFIG_HOME"
+              test ! -w "$XDG_CONFIG_HOME/chromium"
+              # Check the shipped executable before the installer can repair
+              # an image that accidentally omitted this opt-in dependency.
+              "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
+              archivebox install opencode
+              "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
+            '
+
       - name: Validate exact built image version and commit
         if: ${{ !inputs.push_digests }}
         shell: bash
         run: |
           set -Eeuo pipefail
-          IMAGE="${{ matrix.local_tag }}"
-          SHORT_SHA="${GITHUB_SHA::7}"
+          IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}"
+          SHORT_SHA="${{ (inputs.source_ref || github.sha) }}"
+          SHORT_SHA="${SHORT_SHA::7}"
 
           DATA_DIR="$(mktemp -d)"
           "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
@@ -296,8 +345,8 @@ jobs:
             MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${IMAGE_DIGEST}")"
           fi
           COMPRESSED_SIZE="$(MANIFEST="$MANIFEST" "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); print(manifest["config"]["size"] + sum(layer["size"] for layer in manifest["layers"]))')"
-          LIMIT=$((750 * 1024 * 1024))
-          echo "${DIGEST} compressed_size=$((COMPRESSED_SIZE / 1024 / 1024)) MiB limit=750 MiB"
+          LIMIT=$((1025 * 1024 * 1024))
+          echo "${DIGEST} compressed_size=$((COMPRESSED_SIZE / 1024 / 1024)) MiB limit=1025 MiB"
           (( COMPRESSED_SIZE < LIMIT ))
 
       - name: Resolve real Docker CLI test dependencies through abxpkg
@@ -375,14 +424,20 @@ jobs:
 
           NFS_TEST_DIR="$NFS_MOUNT" \
           SMB_TEST_DIR="$SMB_MOUNT" \
-          IMAGE="${{ matrix.local_tag }}" \
+          IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
             "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
 
+      - name: Validate Docker port compatibility
+        shell: bash
+        env:
+          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
+        run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
+
       - name: Validate Docker CLI add/list and compose behavior
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash
         env:
-          ARCHIVEBOX_IMAGE: ${{ matrix.local_tag }}
+          ARCHIVEBOX_IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
         run: |
           set -Eeuo pipefail
           "$MKDIR_BINARY" "${{ github.workspace }}/data"
@@ -401,13 +456,13 @@ jobs:
           "$DOCKER_BINARY" compose run --rm archivebox init
           "$DOCKER_BINARY" compose up -d --wait
           "$DOCKER_BINARY" network connect archivebox_default www-nginx
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/public/')"
-          admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js')"
+          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
+          admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
           [[ "$index_html" == *ArchiveBox* ]]
           [[ "$admin_js" == *window.django* ]]
 
           "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/public/')"
+          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
           [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
           "$DOCKER_BINARY" compose down
 
@@ -423,7 +478,7 @@ jobs:
             '${{ runner.temp }}/archivebox-image.tar' \
             '${{ steps.docker_build_pr.outputs.digest }}' \
             "$LOCAL_IMAGE_ID" \
-            "$((750 * 1024 * 1024))" <<'PY'
+            "$((1025 * 1024 * 1024))" <<'PY'
           from hashlib import sha256
           import json
           import os
@@ -503,6 +558,7 @@ jobs:
           mkdir -p /tmp/digests
           digest="${{ steps.docker_build.outputs.digest }}"
           touch "/tmp/digests/${digest#sha256:}"
+          printf '%s\n' '${{ inputs.source_ref || github.sha }}' > /tmp/digests/source-ref
 
       - name: Upload digest
         if: inputs.push_digests

.github/workflows/docs-versions.yml

@@ -1,33 +0,0 @@
-name: Documentation version policy
-
-on:
-  workflow_run:
-    workflows: [Release]
-    types: [completed]
-  workflow_dispatch:
-  schedule:
-    - cron: '17 * * * *'
-
-permissions:
-  contents: read
-
-concurrency:
-  group: documentation-version-policy
-  cancel-in-progress: false
-
-jobs:
-  reconcile:
-    if: github.repository == 'ArchiveBox/ArchiveBox' && (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success')
-    runs-on: ubuntu-24.04
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          ref: dev
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: '0.11.3'
-      - name: Reconcile published versions without removing historical URLs
-        env:
-          READTHEDOCS_TOKEN: ${{ secrets.READTHEDOCS_TOKEN }}
-        run: uv run --no-project --with 'requests==2.34.2' --with 'packaging==26.2' python bin/sync_docs_versions.py --apply

.github/workflows/docs.yml

@@ -9,38 +9,6 @@ env:
   SHOW_PROGRESS: "False"
 
 jobs:
-  build:
-    name: Build Sphinx site and generated API reference
-    runs-on: ubuntu-24.04
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-      - name: Install documentation build dependencies
-        run: uv sync --locked --dev
-      - name: Build documentation from source, including untracked API pages
-        env:
-          READTHEDOCS: "True"
-          READTHEDOCS_VERSION: ${{ github.head_ref || github.ref_name }}
-          READTHEDOCS_GIT_COMMIT_HASH: ${{ github.sha }}
-        run: uv run --no-sync sphinx-build -W --keep-going -b html docs "$RUNNER_TEMP/docs-html"
-      - name: Verify API navigation and Read the Docs version-selector assets
-        run: |
-          test -s "$RUNNER_TEMP/docs-html/apidocs/index.html"
-          test -s "$RUNNER_TEMP/docs-html/apidocs/archivebox/archivebox.core.models.snapshots.html"
-          test -s "$RUNNER_TEMP/docs-html/apidocs/archivebox/archivebox.api.v1_core.snapshots.html"
-          grep -q 'apidocs/index.html' "$RUNNER_TEMP/docs-html/Contents.html"
-          grep -q 'class="version-switch"' "$RUNNER_TEMP/docs-html/index.html"
-          grep -q 'readthedocs-addons-data-ready' "$RUNNER_TEMP/docs-html/_static/js/versions.js"
-          grep -q 'const themeVersionSelector = true' "$RUNNER_TEMP/docs-html/_static/js/versions.js"
-          ! grep -q 'Edit on GitHub' "$RUNNER_TEMP/docs-html/apidocs/archivebox/archivebox.core.models.snapshots.html"
-          grep -q "github.com/ArchiveBox/ArchiveBox/blob/$GITHUB_SHA/README.md" "$RUNNER_TEMP/docs-html/README.html"
-
   discover:
     name: Discover documentation examples
     runs-on: ubuntu-24.04
@@ -178,7 +146,9 @@ jobs:
           fi
       - name: Build the exact checkout's documentation image
         if: ${{ matrix.environment == 'docker' }}
-        run: '"$DOCKER_BINARY" build --tag archivebox/archivebox:dev .'
+        run: |
+          ABX_DL_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(dep.removeprefix("abx-dl==") for dep in tomllib.load(open("pyproject.toml", "rb"))["project"]["dependencies"] if dep.startswith("abx-dl==")))')"
+          "$DOCKER_BINARY" build --build-arg "ABX_DL_IMAGE=archivebox/abx-dl:${ABX_DL_VERSION}" --tag archivebox/archivebox:dev .
       - name: Run the Ubuntu, Docker, or macOS documentation lane
         if: ${{ !contains(fromJSON('["root", "freebsd", "openbsd"]'), matrix.environment) }}
         run: uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'

.github/workflows/release-candidate.yml

@@ -2,7 +2,7 @@ name: Release candidate
 
 on:
   push:
-    branches: [dev]
+    branches: [dev, main]
 
 permissions:
   actions: read
@@ -10,7 +10,7 @@ permissions:
   packages: write
 
 concurrency:
-  group: release-candidate-dev
+  group: release-candidate-${{ github.ref_name }}
   cancel-in-progress: true
 
 jobs:
@@ -25,17 +25,31 @@ jobs:
           submodules: true
           token: ${{ secrets.RELEASE_GH_TOKEN }}
 
+      - name: Keep synchronized stable branches on the released version
+        id: channel
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
+          MAIN_SHA="$(git ls-remote origin refs/heads/main | cut -f1)"
+          if [[ "$GITHUB_REF_NAME" == dev && "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ && "$MAIN_SHA" == "$GITHUB_SHA" ]]; then
+            echo 'stable=true' >> "$GITHUB_OUTPUT"
+          else
+            echo 'stable=false' >> "$GITHUB_OUTPUT"
+          fi
+
       - id: version
+        if: steps.channel.outputs.stable != 'true'
         uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
         with:
           package: archivebox
-          branch: dev
+          branch: ${{ github.ref_name }}
           tag-prefix: v
-          version-scheme: rc
+          version-scheme: ${{ github.ref_name == 'main' && 'patch' || 'rc' }}
 
   python-artifacts:
     needs: prepare
-    if: needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved'
+    if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
     uses: ./.github/workflows/pip.yml
     with:
       full_tests: false
@@ -43,9 +57,23 @@ jobs:
 
   docker-digests:
     needs: prepare
-    if: needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved'
+    if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
     uses: ./.github/workflows/docker.yml
     with:
       push_digests: true
       full_tests: false
     secrets: inherit
+
+  opencode:
+    needs: prepare
+    if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
+    strategy:
+      fail-fast: false
+      matrix:
+        test_file:
+          - archivebox/tests/test_opencode_agent.py
+          - archivebox/tests/test_opencode_browser.py
+    uses: ./.github/workflows/test-parallel.yml
+    with:
+      test_file: ${{ matrix.test_file }}
+    secrets: inherit

.github/workflows/release.yml

@@ -5,30 +5,53 @@ on:
     workflows: [CI, Release candidate]
     types: [completed]
     branches: [dev, main]
+  workflow_dispatch:
+    inputs:
+      release_branch:
+        description: Branch being promoted
+        required: true
+        type: choice
+        options: [main, dev]
+      release_sha:
+        description: Exact tested commit to release
+        required: true
+        type: string
+      ci_run_id:
+        description: Successful CI run containing the tested Python artifacts
+        required: true
+        type: string
+      docker_digest_run_id:
+        description: Manual Docker digest build run containing the tested image digests
+        required: true
+        type: string
 
 permissions: {}
 
 concurrency:
-  group: release-${{ github.repository }}-${{ github.event.workflow_run.head_branch }}
+  group: release-${{ github.repository }}-${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
   cancel-in-progress: false
 
 env:
   DOCKERHUB_IMAGE: archivebox/archivebox
   GHCR_IMAGE: ghcr.io/archivebox/archivebox
-  RELEASE_BRANCH: ${{ github.event.workflow_run.head_branch }}
-  RELEASE_SHA: ${{ github.event.workflow_run.head_sha }}
-  CI_RUN_ID: ${{ github.event.workflow_run.id }}
+  RELEASE_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
+  RELEASE_SHA: ${{ github.event_name == 'workflow_dispatch' && inputs.release_sha || github.event.workflow_run.head_sha }}
 
 jobs:
   candidate:
     if: >-
       github.event.workflow_run.conclusion == 'success' &&
       github.event.workflow_run.event == 'push' &&
-      ((github.event.workflow_run.head_branch == 'dev' && github.event.workflow_run.name == 'Release candidate') ||
-       (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.name == 'CI'))
+      (github.event.workflow_run.head_branch == 'dev' || github.event.workflow_run.head_branch == 'main') ||
+      (github.event_name == 'workflow_dispatch' && inputs.release_branch == 'main')
     runs-on: ubuntu-24.04
+    permissions:
+      actions: read
+      contents: read
     outputs:
-      ready: ${{ steps.owner.outputs.ready }}
+      ready: ${{ steps.verified.outputs.ready }}
+      artifact_run_id: ${{ steps.verified.outputs.artifact_run_id }}
+      digest_run_id: ${{ steps.verified.outputs.digest_run_id }}
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
@@ -37,19 +60,81 @@ jobs:
       - id: owner
         shell: bash
         run: |
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            echo 'ready=true' >> "$GITHUB_OUTPUT"
+          set -Eeuo pipefail
+          BRANCH_SHA="$(git ls-remote origin "refs/heads/$RELEASE_BRANCH" | cut -f1)"
+          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && "$BRANCH_SHA" != "$RELEASE_SHA" ]]; then
+            echo 'ready=false' >> "$GITHUB_OUTPUT"
             exit 0
           fi
           VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
+          if [[ "$RELEASE_BRANCH" == main ]]; then
+            TAG_TARGET="$(git ls-remote origin "refs/tags/v$VERSION" | cut -f1)"
+            if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && -n "$TAG_TARGET" && "$TAG_TARGET" != "$RELEASE_SHA" ]]; then
+              echo 'ready=false' >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo 'ready=true' >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          # A stable release fast-forwards dev to main. Only main publishes it;
+          # the resulting dev workflow is not a new release candidate.
+          if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]]; then
+            echo 'ready=false' >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
           TARGET="$(git ls-remote origin "refs/tags/release-candidate/$VERSION" | awk '{print $1}')"
           [[ "$TARGET" == "$RELEASE_SHA" ]] && echo 'ready=true' >> "$GITHUB_OUTPUT" || echo 'ready=false' >> "$GITHUB_OUTPUT"
 
+      - name: Require successful full CI for this commit
+        id: verified
+        if: steps.owner.outputs.ready == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          MANUAL_CI_RUN_ID: ${{ inputs.ci_run_id }}
+          MANUAL_DIGEST_RUN_ID: ${{ inputs.docker_digest_run_id }}
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          if [[ "$GITHUB_EVENT_NAME" == workflow_dispatch ]]; then
+            CI_RUN_ID="$MANUAL_CI_RUN_ID"
+            DIGEST_RUN_ID="$MANUAL_DIGEST_RUN_ID"
+            CI_STATUS="$(gh run view "$CI_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status,conclusion,headSha,event,name \
+              --jq '[.status, .conclusion, .headSha, .event, .name] | @tsv')"
+            [[ "$CI_STATUS" == "$(printf 'completed\tsuccess\t%s\tpush\tCI' "$RELEASE_SHA")" ]]
+          else
+            CANDIDATE_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow release-candidate.yml \
+              --commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
+              --json databaseId,status,conclusion \
+              --jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
+            CI_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml \
+              --commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
+              --json databaseId,status,conclusion \
+              --jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
+            [[ -n "$CANDIDATE_RUN_ID" && -n "$CI_RUN_ID" ]] || exit 0
+            if [[ "$RELEASE_BRANCH" == dev ]]; then
+              DIGEST_RUN_ID="$CANDIDATE_RUN_ID"
+            else
+              DIGEST_RUN_ID="$CI_RUN_ID"
+            fi
+          fi
+          if [[ "$RELEASE_BRANCH" == dev && "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
+            ARTIFACT_RUN_ID="$CANDIDATE_RUN_ID"
+          else
+            ARTIFACT_RUN_ID="$CI_RUN_ID"
+          fi
+          {
+            echo 'ready=true'
+            echo "artifact_run_id=$ARTIFACT_RUN_ID"
+            echo "digest_run_id=$DIGEST_RUN_ID"
+          } >> "$GITHUB_OUTPUT"
+
   python-release:
     name: Publish tested Python artifacts
     needs: candidate
     if: needs.candidate.outputs.ready == 'true'
     runs-on: ubuntu-24.04
+    env:
+      CI_RUN_ID: ${{ needs.candidate.outputs.artifact_run_id }}
     environment: pypi
     permissions:
       actions: read
@@ -162,20 +247,37 @@ jobs:
   docker-release:
     name: Publish tested multiarch image
     if: >-
-      github.event.workflow_run.conclusion == 'success' &&
-      github.event.workflow_run.event == 'push'
-    needs: python-release
+      (github.event_name == 'workflow_dispatch') ||
+      (github.event.workflow_run.conclusion == 'success' &&
+       github.event.workflow_run.event == 'push')
+    needs: [candidate, python-release]
     runs-on: ubuntu-24.04
+    env:
+      DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
     permissions:
       actions: read
-      contents: read
+      contents: write
       packages: write
     steps:
       - name: Checkout the tested commit
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
-          fetch-depth: 1
+          fetch-depth: 0
           ref: ${{ env.RELEASE_SHA }}
+          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
+
+      - name: Fast-forward dev to the tested stable release
+        if: env.RELEASE_BRANCH == 'main' && needs.python-release.outputs.release_ready == 'true'
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          git fetch --no-tags origin main dev
+          MAIN_SHA="$(git rev-parse origin/main)"
+          DEV_SHA="$(git rev-parse origin/dev)"
+          if [[ "$MAIN_SHA" == "$RELEASE_SHA" && "$DEV_SHA" != "$RELEASE_SHA" ]] &&
+             git merge-base --is-ancestor "$DEV_SHA" "$RELEASE_SHA"; then
+            git push origin "$RELEASE_SHA:refs/heads/dev"
+          fi
 
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
@@ -198,13 +300,14 @@ jobs:
             --lib="$ABXPKG_LIB_DIR" \
             --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
             --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null
-          for name in uv git jq docker; do
+          for name in uv gh git jq docker; do
             test -L "$ABXPKG_LIB_DIR/env/bin/$name"
             test -x "$ABXPKG_LIB_DIR/env/bin/$name"
           done
           {
             echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
             echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
+            echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
             echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
             echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
           } >> "$GITHUB_ENV"
@@ -232,16 +335,31 @@ jobs:
           merge-multiple: true
           github-token: ${{ github.token }}
           repository: ${{ github.repository }}
-          run-id: ${{ env.CI_RUN_ID }}
+          run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
+
+      - name: Verify exact successful digest build run
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -Eeuo pipefail
+          RUN_STATUS="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status --jq '.status')"
+          RUN_CONCLUSION="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json conclusion --jq '.conclusion')"
+          [[ "$RUN_STATUS" == completed && "$RUN_CONCLUSION" == success ]]
+          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
+            RUN_SHA="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json headSha --jq '.headSha')"
+            [[ "$RUN_SHA" == "$RELEASE_SHA" ]]
+          fi
 
       - name: Verify digest set and collect tags
         id: docker_meta
         shell: bash
         run: |
           set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
           [[ "${#DIGESTS[@]}" -eq 2 ]]
           for digest in "${DIGESTS[@]}"; do [[ "$digest" =~ ^[0-9a-f]{64}$ ]]; done
+          [[ "$(< /tmp/digests/source-ref)" == "$RELEASE_SHA" ]]
 
           VERSION='${{ needs.python-release.outputs.version }}'
           SHORT_SHA="${RELEASE_SHA::12}"
@@ -251,12 +369,25 @@ jobs:
             TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")"
             TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
           fi
+          SYNC_DEV=false
+          if [[ "$RELEASE_BRANCH" == main ]]; then
+            $GIT_BINARY fetch --no-tags origin main dev
+            MAIN_TARGET="$($GIT_BINARY rev-parse origin/main)"
+            DEV_TARGET="$($GIT_BINARY rev-parse origin/dev)"
+            # Recovery can add CI/docs fixes after the immutable release tag.
+            # Both branches and every shipped application file must agree.
+            if [[ "$MAIN_TARGET" == "$DEV_TARGET" ]] &&
+               $GIT_BINARY diff --quiet "$RELEASE_SHA" "$MAIN_TARGET" -- . ':!.github' ':!README.md'; then
+              SYNC_DEV=true
+            fi
+          fi
           {
             echo 'dockerhub_tags<<EOF'
             if [[ "$RELEASE_BRANCH" == dev ]]; then
               echo "${DOCKERHUB_IMAGE}:dev"
             elif [[ "$RELEASE_BRANCH" == main ]]; then
               echo "${DOCKERHUB_IMAGE}:latest"
+              [[ "$SYNC_DEV" == true ]] && echo "${DOCKERHUB_IMAGE}:dev"
             fi
             echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
             [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${DOCKERHUB_IMAGE}:${VERSION}"
@@ -266,6 +397,7 @@ jobs:
               echo "${GHCR_IMAGE}:dev"
             elif [[ "$RELEASE_BRANCH" == main ]]; then
               echo "${GHCR_IMAGE}:latest"
+              [[ "$SYNC_DEV" == true ]] && echo "${GHCR_IMAGE}:dev"
             fi
             echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
             [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${GHCR_IMAGE}:${VERSION}"
@@ -276,7 +408,7 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
           TAG_ARGS=()
           while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
           REFS=()
@@ -287,7 +419,7 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
           TAG_ARGS=()
           while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
           REFS=()
@@ -317,6 +449,7 @@ jobs:
     name: Cascade verified release
     if: needs.python-release.outputs.release_ready == 'true'
     needs:
+      - candidate
       - python-release
       - docker-release
     permissions:
@@ -327,5 +460,6 @@ jobs:
       upstream_package: archivebox
       upstream_version: ${{ needs.python-release.outputs.version }}
       upstream_artifact_name: python-distributions
+      upstream_run_id: ${{ needs.candidate.outputs.artifact_run_id }}
     secrets:
       release_token: ${{ secrets.RELEASE_GH_TOKEN }}

.github/workflows/screenshots.yml

@@ -5,7 +5,6 @@ on:
     branches:
       - dev
     paths:
-      - publicsite/**
       - archivebox/**
       - bin/collect_ui_screenshots.*
       - bin/generate_ui_screenshot_gallery.py
@@ -43,13 +42,14 @@ jobs:
       - name: Build UI screenshot gallery
         run: bin/collect_ui_screenshots.sh
         env:
-          BASE_URL: http://archivebox.localhost:8000
+          BASE_URL: http://archivebox.localhost:5797
 
       - name: Verify gallery was generated from this exact revision
         run: |
           uv run --no-cache --no-sync python - <<'PY'
           import json
           import os
+          import re
           import tomllib
           from pathlib import Path
 
@@ -59,6 +59,7 @@ jobs:
           assert provenance["version"] == version, provenance
           assert provenance["capture_count"] >= 3 * 35, provenance
           assert len(provenance["files"]) == provenance["capture_count"], provenance
+          assert all(not re.match(r"^\d+-", name) for name in provenance["files"]), provenance
           PY
 
       - name: Save validated screenshots

.github/workflows/test-parallel.yml

@@ -2,6 +2,12 @@ name: Parallel Tests
 
 on:
   workflow_call:
+    inputs:
+      test_file:
+        description: Run one discovered test file; empty runs the complete matrix
+        required: false
+        default: ""
+        type: string
 
 env:
   PYTHONIOENCODING: utf-8
@@ -58,10 +64,15 @@ jobs:
 
       - name: Discover every test exactly once
         id: discover
+        env:
+          TEST_FILE: ${{ inputs.test_file }}
         run: |
           set -Eeuo pipefail
           mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
           test "${#output[@]}" -eq 2
+          if [[ -n "$TEST_FILE" ]]; then
+            output[1]="$("$JQ_BINARY" -ce '[.[] | select(.paths == [env.TEST_FILE])] | select(length == 1)' <<< "${output[1]}")"
+          fi
           echo "tests=${output[1]}" >> "$GITHUB_OUTPUT"
           echo "${output[0]}"
           "$JQ_BINARY" -e 'length > 0 and length <= 256 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}"
@@ -132,6 +143,9 @@ jobs:
             extra_args=(--extra "$TEST_EXTRA")
           fi
           uv sync --no-cache --locked --dev "${extra_args[@]}"
+      - name: Install Playwright Chromium for browser tests
+        if: contains(matrix.test.paths_arg, 'test_proof_previews') || contains(matrix.test.paths_arg, 'test_snapshot_output_stacks') || contains(matrix.test.paths_arg, 'test_ui_admin_snapshot')
+        run: uv run --no-cache --no-sync --no-sources playwright install chromium
 
       - name: Run ${{ matrix.test.name }}
         env:

.gitignore

@@ -68,3 +68,5 @@ ArchiveBox.conf
 
 # Local Claude Code task locks (not project state)
 .claude/scheduled_tasks.lock
+
+/_site/

.pre-commit-config.yaml

@@ -56,6 +56,8 @@ repos:
       - id: mixed-line-ending
       - id: fix-byte-order-marker
       - id: end-of-file-fixer
+        # Preserve the exact bytes of captured cryptographic proof fixtures.
+        exclude: ^archivebox/tests/fixtures/legacy_proofs/.*\.json$
       - id: detect-private-key
       - id: debug-statements
       - id: forbid-submodules
@@ -64,4 +66,4 @@ repos:
         args: ["--maxkb=600"]
       - id: name-tests-test
         args: ["--pytest-test-first"]
-        exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$|import_helpers\.py$)
+        exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$)

AGENTS.md

@@ -84,4 +84,4 @@ uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_ta
 uv run prek run --all-files
 ```
 
-Releases are published only by `.github/workflows/release.yml` after the complete `dev` CI workflow succeeds. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.
+Releases are published only by `.github/workflows/release.yml` after complete CI succeeds for the exact commit. Pushes to `main` prepare stable patch releases automatically; `dev` prepares release candidates. Version selection and dependency repinning are owned by CI and the monorepo coordinator. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.

Dockerfile

@@ -1,8 +1,38 @@
 # syntax=docker/dockerfile:1.7
 
 # Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
-# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
-# ArchiveBox owns sonic, supervisor, Django, and the app runtime.
+# WHY: abx-dl owns Python, Node, Chromium, and extraction dependencies so the
+# standalone downloader stays independent of ArchiveBox's server features.
+# ArchiveBox-only dependencies (Sonic, OpenCode, supervisor, Django) belong in
+# the layers added by this Dockerfile, never in the shared abx-dl base image.
+# REQUIRED INSTALLATION CONTRACT — do not work around failures:
+# 1. IMAGE CONSTRUCTION: preinstall ALL plugins supported by runtime="archivebox"
+#    and ALL their required_binaries, regardless of default enabled state.
+#    Inherit the complete downloader dependencies from the tested abx-dl image;
+#    install every remaining ArchiveBox-only dependency in this image's layers,
+#    including OpenCode and Sonic. Never put server dependencies in abx-dl.
+#    Use the normal plugin dependency pipeline: install command -> config.json
+#    required_binaries -> abxpkg. Never add manual apt/pip/npm installs to hide
+#    failures in that pipeline. OS/interpreter bootstrap is a separate concern,
+#    not a second hand-maintained list of plugin dependencies.
+# 2. FINISHED IMAGE: archivebox install and the full plugin installation check
+#    must discover everything already present and be NO-OPS offline as UID 911.
+#    No downloads, package installs, sudo, or root repair may be needed to enable
+#    ANY plugin or start a new crawl. Never run verification as root or manually
+#    preinstall packages before it. If the inherited image lacks a downloader
+#    dependency, fix/release abx-dl and consume that tested image; do not patch
+#    over the missing base dependency here or repair deployed containers by SSH.
+# 3. NON-CONTAINER CI: exercise real runtime auto-installation through the same
+#    CLI/required_binaries/abxpkg path users invoke. GitHub's runner has
+#    passwordless sudo; abxpkg must escalate automatically when needed. Never
+#    manually preinstall dependencies or wrap the tested command in sudo to make
+#    a failure disappear. This is distinct from the finished-image no-op test.
+# 4. SIZE: the complete compressed image budget is 1025 MiB. Preserve ALL
+#    user-facing functionality. Only mechanical/internal
+#    optimizations are allowed. Never remove dependencies, disable features or
+#    plugins, weaken assertions, or skip checks to meet a size limit. If the
+#    complete image exceeds the configured limit, LET CI FAIL and explicitly
+#    ask the maintainer what to do. Do not raise the limit unilaterally.
 # Build abx-dl first, then point this file at it:
 #   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
 #       --build-context abxbus=../abxbus \
@@ -10,10 +40,10 @@
 #       --build-context abx-plugins=../abx-plugins \
 #       -t archivebox/abx-dl:dev
 #   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.289 \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:dev \
 #       -t archivebox:multistage
 
-ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.289
+ARG ABX_DL_IMAGE=archivebox/abx-dl:main
 
 FROM archivebox/sonic:1.4.9 AS sonic
 FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
@@ -47,6 +77,7 @@ ENV ARCHIVEBOX_USER=archivebox \
     IN_DOCKER=True
 
 ENV CODE_DIR=/app \
+    ABX_RUNTIME=archivebox \
     DATA_DIR=/data \
     CONFIG_DIR=/opt/archivebox \
     ABXPKG_LIB_DIR=/opt/archivebox/lib \
@@ -250,6 +281,21 @@ RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777
     && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
     && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
 
+# Install the complete ArchiveBox catalog, including disabled server plugins.
+# Explicit plugin names make installation independent of enabled defaults.
+RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-opencode-$TARGETARCH \
+    chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" /var/tmp/abxpkg-cache \
+    && chmod 1777 /var/tmp/abxpkg-cache \
+    && export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
+    && env ABXPKG_TMP_CACHE_DIR=/var/tmp/abxpkg-cache XDG_CACHE_HOME=/var/tmp/abxpkg-cache \
+        setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install $ABX_DOCKER_PLUGINS \
+    # pnpm includes musl variants that Debian's glibc runtime cannot use.
+    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules" -type l -name 'opencode-linux-*-musl' -delete \
+    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.pnpm" -maxdepth 1 -type d -name 'opencode-linux-*-musl@*' -exec rm -rf {} +
+RUN --network=none export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups \
+    bash -c '"$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version && abx-dl install $ABX_DOCKER_PLUGINS'
+
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
     && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
@@ -282,10 +328,10 @@ RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully
 
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
-EXPOSE 8000
+EXPOSE 5797
 
 HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
+    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:5797/health/' | grep -q 'OK'
 
 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]
+CMD ["archivebox", "server", "--init", "0.0.0.0:5797"]

README.md

@@ -1,10 +1,10 @@
 <div align="center" style="text-align: center; width: 100%">
-<img style="height: 90px" src="https://archivebox.io/icon.png"/>
+<img src="https://archivebox.io/icon.png" height="90px"/>
 <h1>ArchiveBox<br/><sub>Open-source self-hosted web archiving.</sub></h1>
 
 <br/>
 
-▶️ <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart">Quickstart</a> | <a href="https://demo.archivebox.io">Demo</a> | <a href="https://github.com/ArchiveBox/ArchiveBox">GitHub</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Documentation</a> | <a href="https://github.com/ArchiveBox/ArchiveBox#background--motivation">Info & Motivation</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community">Community</a>
+▶️ <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart">Quickstart</a> | <a href="https://demo.archivebox.io">Demo</a> | <a href="https://github.com/ArchiveBox/ArchiveBox">GitHub</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Documentation</a> | <a href="#background--motivation">Info & Motivation</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community">Community</a>
 
 <br/>
 
@@ -73,15 +73,15 @@ mkdir -p ~/archivebox/data && cd ~/archivebox
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
 docker compose pull
 docker compose up -d --wait                                                # initializes new collections automatically
-# open http://admin.archivebox.localhost:8000 to finish setup
+# open http://admin.archivebox.localhost:5797 to finish setup
 # docker compose run --rm archivebox add 'https://example.com'
 # docker compose run --rm archivebox help
 <br/>
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -d --name archivebox -v "$PWD:/data" -p 8000:8000 archivebox/archivebox:dev
-# open http://admin.archivebox.localhost:8000 to finish setup
+docker run -d --name archivebox -v "$PWD:/data" -p 5797:5797 archivebox/archivebox:dev
+# open http://admin.archivebox.localhost:5797 to finish setup
 # docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
 # docker run -it -v $PWD:/data archivebox/archivebox:dev help
 <br/>
@@ -93,14 +93,14 @@ archivebox init
 archivebox install
 # archivebox add 'https://example.com'
 # archivebox help
-# archivebox server 0.0.0.0:8000
+# archivebox server 0.0.0.0:5797
 <br/>
 <br/>
 # Option D: Or use the uv install shortcut for Option C
 curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
-<sub>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sub><br/>
+<sub>Open <a href="http://web.archivebox.localhost:5797"><code>http://web.archivebox.localhost:5797</code></a> for the public UI and <a href="http://admin.archivebox.localhost:5797"><code>http://admin.archivebox.localhost:5797</code></a> for the admin UI ➡️</sub><br/>
 <sub>Set <code>BASE_URL</code> to change the public base domain. The default <code>auto</code> mode uses <code>web.</code> and <code>admin.</code> subdomains on <code>*.localhost</code>, but one host for ordinary DNS names. <code>BIND_ADDR</code> only controls the local listen address.</sub>
 </details>
 <br/>
@@ -108,7 +108,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 
 <div align="center" style="text-align: center">
 <br/><br/>
-<img style="height: 70px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" alt="bookshelf graphic"> &nbsp; <img style="height: 75px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/b2765a33-0d1e-4019-a1db-920c7e00e20e" alt="logo" align="top"/> &nbsp; <img style="height: 70px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" alt="bookshelf graphic">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic"> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/b2765a33-0d1e-4019-a1db-920c7e00e20e" height="75px" alt="logo" align="top"/> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic">
 <br/><br/>
 <small><a href="https://demo.archivebox.io">Demo</a> | <a href="#screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
 <br/>
@@ -129,10 +129,10 @@ curl -fsSL 'https://get.archivebox.io' | bash
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
 - [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
-- Advanced users: support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
-- Planned: support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
+- Support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
+- Support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [infiniscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
 
 <br/>
 
@@ -152,18 +152,18 @@ curl -fsSL 'https://get.archivebox.io' | bash
 #### ✳️&nbsp; Easy Setup
 
 <details>
-<summary><b><img style="height: 28px" src="https://archivebox.github.io/ios-archivebox/docs/icons/apple.svg" alt="Apple"  align="top"/> <code>ArchiveBox Server.app</code></b> (macOS)</summary>
+<summary><b><img src="https://app.archivebox.io/docs/icons/apple.svg" alt="Apple" height="28px" align="top"/> <code>ArchiveBox Server.app</code></b> (macOS)</summary>
 <br/>
 <ol>
 <li><a href="https://github.com/ArchiveBox/ios-archivebox/releases">Download <code>ArchiveBox Server.app</code></a>, move it to Applications, and open it.</li>
 <li>Set up an admin account and generate an API Key.</li>
-<li>Use the Web UI or <a href="https://archivebox.github.io/ios-archivebox/"><code>ArchiveBox.app</code></a>, and optionally connect REST API clients, the <a href="https://archivebox.github.io/ios-archivebox/">mobile app</a>, the <a href="https://archivebox.github.io/archivebox-browser-extension/">browser extension</a>, and other integrations to your server using your API Key.</li>
+<li>Use the Web UI or <a href="https://app.archivebox.io/"><code>ArchiveBox.app</code></a>, and optionally connect REST API clients, the <a href="https://app.archivebox.io/">mobile app</a>, the <a href="https://extension.archivebox.io/">browser extension</a>, and other integrations to your server using your API Key.</li>
 </ol>
 <br/>
 </details>
 
 <details>
-<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker"  align="top"/> <code>docker-compose</code></b>  (macOS/Linux) &nbsp; <b>👈&nbsp; recommended</b> &nbsp; <i>(click to expand)</i></summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker-compose</code></b>  (macOS/Linux) &nbsp; <b>👈&nbsp; recommended</b> &nbsp; <i>(click to expand)</i></summary>
 <br/>
 <i>👍 Docker Compose is recommended for the easiest install/update UX + best security + all <a href="#dependencies">extras</a> out-of-the-box.</i>
 <br/><br/>
@@ -177,7 +177,7 @@ docker compose pull
 <li>Start the server, which initializes a new collection automatically.
 <pre lang="bash"><code style="white-space: pre-line">docker compose up -d --wait
 </code></pre></li>
-<li>Open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
+<li>Open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:5797/admin/">http://admin.archivebox.localhost:5797/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
 <pre lang="bash"><code style="white-space: pre-line">
 # run CLI commands inside the server container started above
 docker compose exec archivebox archivebox add 'https://example.com'
@@ -187,21 +187,21 @@ docker compose exec archivebox archivebox help
 </li>
 </ol>
 
-See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage">filesystem/SQL/Python</a> to manage your archive.
+See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage">filesystem/SQL/Python</a> to manage your archive.
 <br/><br/>
 </details>
 
 <details>
-<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <code>docker run</code></b>  (macOS/Linux)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker run</code></b>  (macOS/Linux)</summary>
 <br/>
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
 <li>Create a new empty directory and start the server, which initializes the collection automatically (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -d --name archivebox -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
+docker run -d --name archivebox -v $PWD:/data -p 5797:5797 archivebox/archivebox:dev
 </code></pre>
 </li>
-<li>Open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
+<li>Open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:5797/admin/">http://admin.archivebox.localhost:5797/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
 <pre lang="bash"><code style="white-space: pre-line">
 # completely optional, CLI can always be used without running a server
 # docker exec archivebox archivebox [subcommand] [--help]
@@ -211,12 +211,12 @@ docker exec archivebox archivebox help
 </li>
 </ol>
 
-See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.
+See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.
 <br/><br/>
 </details>
 
 <details>
-<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh uv install shortcut" align="top"/> <code>bash</code> uv install shortcut</b>  (macOS/Linux/BSD)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh uv install shortcut" height="28px" align="top"/> <code>bash</code> uv install shortcut</b>  (macOS/Linux/BSD)</summary>
 <br/>
 <ol>
 <li>Run the shortcut for the <code>uv</code> install method. It installs <code>uv</code> first when needed.
@@ -225,7 +225,7 @@ See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or
 </li>
 </ol>
 
-See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
+See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
 See <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/bin/setup.sh"><code>setup.sh</code></a> for the source code of the <code>uv</code> install shortcut.<br/>
 See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as an install method"</a> blog post for my thoughts on the shortcomings of this install method.
 <br/><br/>
@@ -239,7 +239,7 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a
 
 
 <details>
-<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="uv" align="top"/> <code>uv</code></b> (macOS/Linux/BSD)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="uv" height="28px" align="top"/> <code>uv</code></b> (macOS/Linux/BSD)</summary>
 <br/>
 <ol>
 
@@ -256,8 +256,8 @@ archivebox init     # initialize a new collection
 archivebox install  # install all the runtime dependencies (e.g. chrome, single-file, yt-dlp, etc.)
 </code></pre>
 </li>
-<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
-<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
+<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:5797/admin/">http://admin.archivebox.localhost:5797/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
+<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:5797
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
@@ -265,7 +265,7 @@ archivebox help
 </li>
 </ol>
 
-See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
+See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
 <br/>
 <sub>See the <a href="https://docs.astral.sh/uv/guides/tools/"><code>uv tool</code> documentation</a> for more details about this installation method.</sub>
 <br/><br/>
@@ -273,7 +273,7 @@ See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or
 
 
 <details>
-<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
 <ol>
 <li>Add the third-party ArchiveBox apt repo and install <code>archivebox</code>.
@@ -292,21 +292,21 @@ archivebox add 'https://example.com'
 </code></pre>
 <br/>
 </li>
-<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
-<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
+<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:5797/admin/">http://admin.archivebox.localhost:5797/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
+<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:5797
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
 </code></pre>
 </li>
 </ol>
-See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
+See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
 <sub>The apt package is a thin dev-channel wrapper around the normal Python install flow. <code>sudo archivebox install</code> uses apt for missing system dependencies while preserving ownership of the user-owned collection. See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for details.</sub>
 <br/><br/>
 </details>
 
 <details>
-<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447803-f2ec3700-af0b-11eb-87d3-671d114f011d.png" alt="homebrew" align="top"/> <code>brew</code></b> (macOS and Linux)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447803-f2ec3700-af0b-11eb-87d3-671d114f011d.png" alt="homebrew" height="28px" align="top"/> <code>brew</code></b> (macOS and Linux)</summary>
 <br/>
 Run Homebrew as your normal non-root user on both macOS and Linux; do not use <code>sudo brew</code>.
 <ol>
@@ -325,8 +325,8 @@ archivebox init
 archivebox install
 </code></pre>
 </li>
-<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
-<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
+<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:5797/admin/">http://admin.archivebox.localhost:5797/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
+<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:5797
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
@@ -334,13 +334,13 @@ archivebox help
 </li>
 </ol>
 
-See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
+See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
 <sub>See the <a href="https://github.com/ArchiveBox/homebrew-archivebox"><code>homebrew-archivebox</code></a> repo for more details about this distribution.</sub>
 <br/><br/>
 </details>
 
 <details>
-<summary><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/118077361-f0616580-b381-11eb-973c-ee894a3349fb.png" alt="Arch" align="top"/> <code>pacman</code> / <img style="height: 28px" src="https://user-images.githubusercontent.com/511499/118077946-29e6a080-b383-11eb-94f0-d4871da08c3f.png" alt="FreeBSD" align="top"/> <code>pkg</code> / <img style="height: 28px" src="https://user-images.githubusercontent.com/511499/118077861-002d7980-b383-11eb-86a7-5936fad9190f.png" alt="Nix" align="top"/> <code>nix</code> (Arch/FreeBSD/NixOS/more)</summary>
+<summary><img src="https://user-images.githubusercontent.com/511499/118077361-f0616580-b381-11eb-973c-ee894a3349fb.png" alt="Arch" height="28px" align="top"/> <code>pacman</code> / <img src="https://user-images.githubusercontent.com/511499/118077946-29e6a080-b383-11eb-94f0-d4871da08c3f.png" alt="FreeBSD" height="28px" align="top"/> <code>pkg</code> / <img src="https://user-images.githubusercontent.com/511499/118077861-002d7980-b383-11eb-86a7-5936fad9190f.png" alt="Nix" height="28px" align="top"/> <code>nix</code> (Arch/FreeBSD/NixOS/more)</summary>
 <br/>
 
 > *Warning: These are contributed by external volunteers and may lag behind the official `uv` and Docker channels.*
@@ -352,7 +352,7 @@ See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or
 <li>Guix: <a href="https://packages.guix.gnu.org/packages/archivebox/"><code>guix install archivebox</code></a> (contributed by <a href="https://github.com/rakino"><code>@rakino</code></a>)</li>
 <li>More: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/new"><i>contribute another distribution...!</i></a></li>
 </ul>
-See <a href="#usage">below</a> for usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.
+See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.
 <br/><br/>
 </details>
 
@@ -361,26 +361,21 @@ See <a href="#usage">below</a> for usage examples using the CLI, Web UI, or file
 #### 🎗&nbsp; Other Options
 
 <details>
-<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <code>docker</code> + <img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447263-4316c980-af0b-11eb-928d-eaf1292ac646.png" alt="Electron" align="top"/> <code>electron</code> Desktop App</b> (macOS/Linux/Windows)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker</code> + <img src="https://user-images.githubusercontent.com/511499/117447263-4316c980-af0b-11eb-928d-eaf1292ac646.png" alt="Electron" height="28px" align="top"/> <code>electron</code> Desktop App</b> (macOS/Linux/Windows)</summary>
 <br/>
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
-<li>Download a binary release for your OS or build the native app from source<br/>
-<ul>
-<li>macOS: <a href="https://github.com/ArchiveBox/ArchiveBox/releases/download/v0.6.2/Electron-ArchiveBox-macOS-x64-0.6.2.app.zip" download><code>ArchiveBox.app.zip</code></a></li>
-<li>Linux: <code>ArchiveBox.deb</code> (alpha: <a href="https://github.com/ArchiveBox/electron-archivebox#quickstart">build manually</a>)</li>
-<li>Windows: <code>ArchiveBox.exe</code> (beta: <a href="https://github.com/ArchiveBox/electron-archivebox#quickstart">build manually</a>)</li>
-</ul>
-</li>
+<li><a href="https://electron.archivebox.io/#get-started">Download the latest installer for Windows, Linux, or Mac</a>.</li>
+<li>Install and open ArchiveBox Desktop, then create your local administrator account to start archiving.</li>
 </ol>
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/575ef92f-bb3e-4a7c-a4ba-986c1fd76ecf" width="320px">
 <br/>
-<i>✨ Alpha (contributors wanted!)</i>: for more info, see the: <a href="https://github.com/ArchiveBox/electron-archivebox">Electron ArchiveBox</a> repo.
+<a href="https://electron.archivebox.io/">Desktop app guide</a> · <a href="https://electron.archivebox.io/screenshots/">Screenshots</a>
 <br/>
 </details>
 
 <details>
-<summary><img style="height: 28px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0c46e949-00fe-49c8-a613-ee14501c014c" alt="Self-hosting Platforms" align="top"/><b> TrueNAS / UNRAID / YunoHost / Cloudron / etc.</b> (self-hosting solutions)</summary>
+<summary><img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0c46e949-00fe-49c8-a613-ee14501c014c" alt="Self-hosting Platforms" height="28px" align="top"/><b> TrueNAS / UNRAID / YunoHost / Cloudron / etc.</b> (self-hosting solutions)</summary>
 <br/>
 
 > *Warning: These are contributed by external volunteers and may lag behind the official Docker and `uv` channels.*
@@ -399,31 +394,31 @@ See <a href="#usage">below</a> for usage examples using the CLI, Web UI, or file
 
 <li>More: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/new"><i>contribute another distribution...!</i></a></li>
 </ul>
-See <a href="#usage">below</a> for usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.
+See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.
 <br/><br/>
 </details>
 
 <details>
-<summary><img style="height: 27px" src="https://user-images.githubusercontent.com/511499/117448723-1663b180-af0d-11eb-837f-d43959227810.png" alt="paid" align="top"/> Third-party paid hosting options (cloud VPS)</summary>
+<summary><img src="https://user-images.githubusercontent.com/511499/117448723-1663b180-af0d-11eb-837f-d43959227810.png" alt="paid" height="27px" align="top"/> Third-party paid hosting options (cloud VPS)</summary>
 <br/>
 <ul>
-<li><a href="https://elest.io/open-source/archivebox"><img style="height: 22px" src="https://img.shields.io/badge/Managed_Hosting-Elest.io-%23193f7e.svg?style=flat"/></a></li>
-<li><a href="https://www.stellarhosted.com/archivebox/"><img style="height: 22px" src="https://img.shields.io/badge/Semi_Managed_Hosting-StellarHosted.com-%23193f7e.svg?style=flat"/></a> (USD $29-250/mo, <a href="https://www.stellarhosted.com/archivebox/#pricing">pricing</a>)</li>
-<li><a href="https://www.pikapods.com/pods?run=archivebox"><img style="height: 22px" src="https://img.shields.io/badge/Semi_Managed_Hosting-PikaPods.com-%2343a047.svg?style=flat"/></a> (from USD $2.6/mo)</li>
+<li><a href="https://elest.io/open-source/archivebox"><img src="https://img.shields.io/badge/Managed_Hosting-Elest.io-%23193f7e.svg?style=flat" height="22px"/></a></li>
+<li><a href="https://www.stellarhosted.com/archivebox/"><img src="https://img.shields.io/badge/Semi_Managed_Hosting-StellarHosted.com-%23193f7e.svg?style=flat" height="22px"/></a> (USD $29-250/mo, <a href="https://www.stellarhosted.com/archivebox/#pricing">pricing</a>)</li>
+<li><a href="https://www.pikapods.com/pods?run=archivebox"><img src="https://img.shields.io/badge/Semi_Managed_Hosting-PikaPods.com-%2343a047.svg?style=flat" height="22px"/></a> (from USD $2.6/mo)</li>
 <li><a href="https://m.do.co/c/cbc4c0c17840">
- <img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-DigitalOcean.com-%232f7cf7.svg?style=flat"/>
+ <img src="https://img.shields.io/badge/Unmanaged_VPS-DigitalOcean.com-%232f7cf7.svg?style=flat" height="22px"/>
 </a> (USD $5-50+/mo, <a href="https://m.do.co/c/cbc4c0c17840">🎗&nbsp; referral link</a>, <a href="https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-compose-on-ubuntu-20-04">instructions</a>)</li>
 <li><a href="https://www.vultr.com/?ref=7130289">
- <img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-Vultr.com-%232337a8.svg?style=flat"/>
+ <img src="https://img.shields.io/badge/Unmanaged_VPS-Vultr.com-%232337a8.svg?style=flat" height="22px"/>
 </a> (USD $2.5-50+/mo, <a href="https://www.vultr.com/?ref=7130289">🎗&nbsp; referral link</a>, <a href="https://www.vultr.com/docs/install-docker-compose-on-ubuntu-20-04">instructions</a>)</li>
 <li><a href="https://fly.io/">
- <img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_App-Fly.io-%239a2de6.svg?style=flat"/>
+ <img src="https://img.shields.io/badge/Unmanaged_App-Fly.io-%239a2de6.svg?style=flat" height="22px"/>
 </a> (USD $10-50+/mo, <a href="https://fly.io/docs/hands-on/start/">instructions</a>)</li>
 <li><a href="https://railway.app/template/2Vvhmy">
- <img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_App-Railway-%23A11BE6.svg?style=flat"/>
+ <img src="https://img.shields.io/badge/Unmanaged_App-Railway-%23A11BE6.svg?style=flat" height="22px"/>
 </a> (USD $0-5+/mo)</li>
-<li><a href="https://aws.amazon.com/marketplace/pp/Linnovate-Open-Source-Innovation-Support-For-Archi/B08RVW6MJ2"><img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-AWS-%23ee8135.svg?style=flat"/></a> (USD $60-200+/mo)</li>
-<li><a href="https://azuremarketplace.microsoft.com/en-us/marketplace/apps/meanio.archivebox?ocid=gtmrewards_whatsnewblog_archivebox_vol118"><img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-Azure-%237cb300.svg?style=flat"/></a> (USD $60-200+/mo)</li>
+<li><a href="https://aws.amazon.com/marketplace/pp/Linnovate-Open-Source-Innovation-Support-For-Archi/B08RVW6MJ2"><img src="https://img.shields.io/badge/Unmanaged_VPS-AWS-%23ee8135.svg?style=flat" height="22px"/></a> (USD $60-200+/mo)</li>
+<li><a href="https://azuremarketplace.microsoft.com/en-us/marketplace/apps/meanio.archivebox?ocid=gtmrewards_whatsnewblog_archivebox_vol118"><img src="https://img.shields.io/badge/Unmanaged_VPS-Azure-%237cb300.svg?style=flat" height="22px"/></a> (USD $60-200+/mo)</li>
 <br/>
 <sub><i>Referral links marked 🎗 provide $5-10 of free credit for new users and help pay for our <a href="https://demo.archivebox.io">demo server</a> hosting costs.</i></sub>
 </ul>
@@ -437,7 +432,7 @@ For more discussion on third-party hosting options see here: <a href="https://gi
 #### ➡️&nbsp; Next Steps
 
 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
-- (Optional) [Import browser cookies and settings into a persona](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile).
+- (Optional) [Import browser cookies and settings into a persona](docs/Chromium-Install.md#import-an-existing-browser-profile).
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -476,7 +471,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev help
 
 <br/>
 <details>
-<summary><img style="height: 22px" src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" align="top"/> <b>CLI Usage Examples: non-Docker</b></summary>
+<summary><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" height="22px" align="top"/> <b>CLI Usage Examples: non-Docker</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you have uv-installed ArchiveBox and it's available in your $PATH first
@@ -495,7 +490,7 @@ archivebox help              # get list of archivebox subcommands that can be ru
 <br/>
 
 <details>
-<summary><img style="height: 22px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <b>CLI Usage Examples: Docker Compose</b></summary>
+<summary><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="22px" align="top"/> <b>CLI Usage Examples: Docker Compose</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
@@ -512,7 +507,7 @@ docker compose exec archivebox archivebox add 'https://example.com'
 <br/>
 
 <details>
-<summary><img style="height: 22px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <b>CLI Usage Examples: Docker</b></summary>
+<summary><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="22px" align="top"/> <b>CLI Usage Examples: Docker</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure the `archivebox` server container from the Quickstart is running first
@@ -544,16 +539,16 @@ find ./archive/users -path '*/snapshots/*/*/*/index.html'  # inspect snapshot da
 <summary><b>🖥&nbsp; Web UI & API Usage</b></summary>
 <pre lang="bash"><code style="white-space: pre-line">
 # Start the server on bare metal (uv/apt/brew):
-archivebox server 0.0.0.0:8000                 # start the server
+archivebox server 0.0.0.0:5797                 # start the server
 <br/>
 # Or with Docker Compose:
 docker compose up                              # start the server
 <br/>
 # Or with a Docker container:
-docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
+docker run -v $PWD:/data -it -p 5797:5797 archivebox/archivebox:dev
 </code></pre>
 
-<sup>Open <a href="http://admin.archivebox.localhost:8000/admin/"><code>http://admin.archivebox.localhost:8000/admin/</code></a> to create the first admin and finish web setup. Use <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI. ➡️</sup><br/>
+<sup>Open <a href="http://admin.archivebox.localhost:5797/admin/"><code>http://admin.archivebox.localhost:5797/admin/</code></a> to create the first admin and finish web setup. Use <a href="http://web.archivebox.localhost:5797"><code>http://web.archivebox.localhost:5797</code></a> for the public UI. ➡️</sup><br/>
 <sup>Advanced: <code>archivebox manage createsuperuser</code> remains available for creating accounts from the CLI.</sup><br/>
 <sup>Set <code>BASE_URL</code> to change the public base domain. The default <code>auto</code> mode uses <code>web.</code> and <code>admin.</code> subdomains on <code>*.localhost</code>, but one host for ordinary DNS names. <code>BIND_ADDR</code> only controls the local listen address.</sup>
 <br/><br/>
@@ -570,18 +565,20 @@ docker compose run --rm archivebox config --set PERMISSIONS=public
 
 # restart the server to apply any config changes
 </code></pre>
+
+Snapshot caching follows each snapshot's saved `PERMISSIONS`, including inherited defaults and snapshot overrides. Public content can be cached by browsers and shared proxies/CDNs; private and unlisted content uses `Cache-Control: private` so only the user's browser can cache it. Both retain normal artifact cache lifetimes and validators for fast repeat viewing. Authentication handoffs are not stored. Changing permissions does not revoke copies already cached under the previous policy; purge any affected shared cache when needed.
 </details>
 
 <br/>
 
 <details>
-<summary><b><img style="height: 22px" src="https://archivebox.github.io/ios-archivebox/docs/icons/apple.svg" alt="Apple"  align="top"/> <img style="height: 22px" src="https://cdn.jsdelivr.net/gh/devicons/devicon@v2.17.0/icons/windows11/windows11-original.svg" alt="Windows"  align="top"/> 🐧&nbsp; Desktop / Mobile / Extension Usage</b></summary>
+<summary><b><img src="https://app.archivebox.io/docs/icons/apple.svg" alt="Apple" height="22px" align="top"/> <img src="https://cdn.jsdelivr.net/gh/devicons/devicon@v2.17.0/icons/windows11/windows11-original.svg" alt="Windows" height="22px" align="top"/> 🐧&nbsp; Desktop / Mobile / Extension Usage</b></summary>
 <br/>
 
-- 📱 [<code>ArchiveBox</code> iOS Mobile App](https://archivebox.github.io/ios-archivebox/)
-- 🖥️ [<code>ArchiveBox.app</code> macOS Desktop App](https://archivebox.github.io/ios-archivebox/)
-- 🧩 [ArchiveBox Browser Extension](https://archivebox.github.io/archivebox-browser-extension/)
-- 💻 [Linux/Windows Desktop App (Requires Docker Desktop)](https://github.com/ArchiveBox/electron-archivebox)
+- 📱 [<code>ArchiveBox</code> iOS Mobile App](https://app.archivebox.io/)
+- 🖥️ [<code>ArchiveBox.app</code> macOS Desktop App](https://app.archivebox.io/)
+- 🧩 [ArchiveBox Browser Extension](https://extension.archivebox.io/)
+- 💻 [Linux/Windows Desktop App (Requires Docker Desktop)](https://electron.archivebox.io/)
 
 <br/>
 </details>
@@ -617,7 +614,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
 <sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
 <br/><br/>
 <a href="https://demo.archivebox.io">DEMO: <code>https://demo.archivebox.io</code></a><br/>
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#caveats">Caveats</a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#Caveats">Caveats</a>
 <br/>
 </div>
 
@@ -638,22 +635,22 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
 ##  Input Formats: How to pass URLs into ArchiveBox for saving
 
 
-- <img style="height: 28px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ff20d251-5347-4b85-ae9b-83037d0ac01e" /> <b>From the official <a href="https://github.com/ArchiveBox/archivebox-extension">ArchiveBox Browser Extension</a></b>  
+- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ff20d251-5347-4b85-ae9b-83037d0ac01e" height="28px"/> <b>From the official <a href="https://github.com/ArchiveBox/archivebox-extension">ArchiveBox Browser Extension</a></b>  
   <i>Provides realtime archiving of browsing history or selected pages from Chrome/Chromium/Firefox/Safari browsers.</i>
 
-- <img style="height: 28px" src="https://archivebox.github.io/ios-archivebox/App/AppIcon.icon/Assets/ArchiveBox.png" alt="ArchiveBox app icon" /> <b>From the official <a href="https://archivebox.github.io/ios-archivebox/">ArchiveBox macOS Desktop App or iOS Mobile App</a></b><br/>
+- <img src="https://app.archivebox.io/App/AppIcon.icon/Assets/ArchiveBox.png" alt="ArchiveBox app icon" height="28px"/> <b>From the official <a href="https://app.archivebox.io/">ArchiveBox macOS Desktop App or iOS Mobile App</a></b><br/>
   <i>Save directly from the Share menu or Siri/Shortcuts to your server.</i>
 
-- <img style="height: 22px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files<br>
+- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8" height="22px"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files  
   <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file).</i>
 
-- <img style="height: 22px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)<br>
+- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34" height="22px"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)  
   <i>Instructions: <a href="https://support.google.com/chrome/answer/96816?hl=en">Chrome</a>, <a href="https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer">Firefox</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/assets/511499/24ad068e-0fa6-41f4-a7ff-4c26fc91f71a">Safari</a>, <a href="https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows">IE</a>, <a href="https://help.opera.com/en/latest/features/#bookmarks:~:text=Click%20the%20import/-,export%20button,-on%20the%20bottom">Opera</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
 
-- <img style="height: 22px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/4f7bd318-265c-4235-ad25-38be89946b12"/> From URLs visited through a [MITM Proxy](https://mitmproxy.org/) with [`archivebox-proxy`](https://github.com/ArchiveBox/archivebox-proxy)<br>
+- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/4f7bd318-265c-4235-ad25-38be89946b12" height="22px"/> From URLs visited through a [MITM Proxy](https://mitmproxy.org/) with [`archivebox-proxy`](https://github.com/ArchiveBox/archivebox-proxy)  
   <i>Provides [realtime archiving](https://github.com/ArchiveBox/ArchiveBox/issues/577) of all traffic from any device going through the proxy.</i>
 
-- <img style="height: 22px" src="https://getpocket.com/favicon.ico"/> From bookmarking services or social media (e.g. Twitter bookmarks, Reddit saved posts, etc.)<br>
+- <img src="https://getpocket.com/favicon.ico" height="22px"/> From bookmarking services or social media (e.g. Twitter bookmarks, Reddit saved posts, etc.)  
   <i>Instructions: <a href="https://getpocket.com/export">Pocket</a>, <a href="https://pinboard.in/export/">Pinboard</a>, <a href="https://www.instapaper.com/user">Instapaper</a>, <a href="https://shaarli.readthedocs.io/en/master/Usage/#importexport">Shaarli</a>, <a href="https://www.groovypost.com/howto/howto/export-delicious-bookmarks-xml/">Delicious</a>, <a href="https://github.com/csu/export-saved-reddit">Reddit Saved</a>, <a href="https://doc.wallabag.org/en/user/import/wallabagv2.html">Wallabag</a>, <a href="http://help.unmark.it/import-export">Unmark.it</a>, <a href="https://www.addictivetips.com/web/onetab-save-close-all-chrome-tabs-to-restore-export-or-import/">OneTab</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/issues/648">Firefox Sync</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
 
 
@@ -1008,14 +1005,11 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active
 
 
 <ul>
-<li>Set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#curl_user_agent"><code>CHROME_USER_AGENT</code>, <code>WGET_USER_AGENT</code>, <code>CURL_USER_AGENT</code></a> to impersonate a real browser (by default, ArchiveBox reveals that it's a bot when using the default user agent settings)</li>
-<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">importing a dedicated browser profile into a persona</a></li>
+<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
 <li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
 
-In the future we plan on adding support for running JS scripts during archiving to block ads, cookie popups, modals, and fix other issues. Follow here for progress: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/51">Issue #51</a>.
-
 </details>
 <br/>
 
@@ -1037,7 +1031,7 @@ Each re-archive creates a distinct Snapshot row for the same URL:
 archivebox add --no-only-new 'https://example.com'
 </code></pre>
 
-The <img style="height: 24px" src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button"/> button in the Admin UI performs the same explicit re-archive.
+The <img src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button" height="24px"/> button in the Admin UI performs the same explicit re-archive.
 
 <h4>Learn More</h4>
 
@@ -1346,25 +1340,25 @@ archivebox init
 archivebox install         # detect and install all extractor dependencies
 
 # Run the development server w/ autoreloading (but no bg workers)
-archivebox server --debug --reload 0.0.0.0:8000
+archivebox server --debug --reload 0.0.0.0:5797
 
 # Run the production server (with bg workers but no autoreloading)
-archivebox server 0.0.0.0:8000
+archivebox server 0.0.0.0:5797
 ```
 
 #### 2. Option B: Build the docker container and use that for development instead
 
 ```bash
 # Optional: develop via docker by mounting the code dir into the container
-# if you edit e.g. ./archivebox/core/models/snapshots.py on the docker host, runserver
+# if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
 # inside the container will reload and pick up your changes
 ./bin/build_docker.sh dev
 
 # Run the development server w/ autoreloading (but no bg workers)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init --debug --reload 0.0.0.0:8000
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 5797:5797 archivebox/archivebox:dev server --init --debug --reload 0.0.0.0:5797
 
 # Run the production server (with bg workers but no autoreloading)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 5797:5797 archivebox/archivebox:dev server --init
 
 # (remove the --reload flag and add the --nothreading flag when profiling with the django debug toolbar)
 # When using --reload, make sure any files you create can be read by the user in the Docker container, eg with 'chmod a+rX'.
@@ -1386,11 +1380,11 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 archivebox config --set DEBUG=True
 
 # OR you can run a dev server with DEBUG=True in a few ways:
-archivebox server --debug --reload 0.0.0.0:8000
+archivebox server --debug --reload 0.0.0.0:5797
 # or
-archivebox server --debug 0.0.0.0:8000
+archivebox server --debug 0.0.0.0:5797
 # or
-env DEBUG=True daphne -b 0.0.0.0 -p 8000 archivebox.core.asgi:application
+env DEBUG=True daphne -b 0.0.0.0 -p 5797 archivebox.core.asgi:application
 ```
 
 https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running
@@ -1425,7 +1419,7 @@ services:
 
 # or with plain Docker:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
-docker run -it -v $PWD:/data -p 8000:8000 archivebox:dev
+docker run -it -v $PWD:/data -p 5797:5797 archivebox:dev
 
 # or with uv:
 uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'

archivebox/api/admin.py

@@ -8,7 +8,7 @@ from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
 from signal_webhooks.settings import webhook_settings
 from signal_webhooks.utils import get_webhook_model, model_from_reference
 
-from archivebox.base_models.admin import card_fieldset, BaseModelAdmin
+from archivebox.base_models.admin import BaseModelAdmin
 
 from archivebox.api.models import APIToken
 
@@ -25,9 +25,27 @@ class APITokenAdmin(BaseModelAdmin):
     search_fields = ("id", "created_by__username", "token")
 
     fieldsets = (
-        card_fieldset("Token", ("token", "expires")),
-        card_fieldset("Owner", ("created_by",)),
-        card_fieldset("Timestamps", ("created_at", "modified_at")),
+        (
+            "Token",
+            {
+                "fields": ("token", "expires"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Owner",
+            {
+                "fields": ("created_by",),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": ("created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
     )
 
     list_filter = ("created_by",)
@@ -55,11 +73,41 @@ class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
     readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)
 
     fieldsets = (
-        card_fieldset("Webhook", _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"), wide=True),
-        card_fieldset("Authentication", _webhook_fields("auth_token")),
-        card_fieldset("Status", _webhook_fields("enabled", "last_success", "last_failure", "last_response")),
-        card_fieldset("Owner", _webhook_fields("created_by")),
-        card_fieldset("Timestamps", _webhook_fields("created_at", "modified_at")),
+        (
+            "Webhook",
+            {
+                "fields": _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Authentication",
+            {
+                "fields": _webhook_fields("auth_token"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Status",
+            {
+                "fields": _webhook_fields("enabled", "last_success", "last_failure", "last_response"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Owner",
+            {
+                "fields": _webhook_fields("created_by"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": _webhook_fields("created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
     )
 
     def lookup_allowed(self, lookup: str, value: str, request: HttpRequest | None = None) -> bool:

archivebox/api/schemas.py

@@ -1,17 +0,0 @@
-"""Fields shared by responses for user-owned model objects."""
-
-from ninja import Schema
-
-
-class OwnedObjectSchema(Schema):
-    created_by_id: str
-    created_by_username: str
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by.pk)
-
-    @staticmethod
-    def resolve_created_by_username(obj) -> str:
-        user = obj.created_by
-        return user.username if isinstance(user.username, str) else str(user)

archivebox/api/v1_cli.py

@@ -117,27 +117,16 @@ def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_
     return kwargs
 
 
-def cli_response(request, result, *, result_format="json", success=True, errors=()):
-    """Use one response envelope for CLI-backed endpoints, including captured output."""
-    payload = {"success": success, "errors": list(errors), "result": result, "result_format": result_format}
-    for stream_name in ("stdout", "stderr"):
-        stream = request.__dict__.get(stream_name)
-        payload[stream_name] = ansi_to_html(stream.getvalue().strip()) if isinstance(stream, StringIO) else ""
-    return payload
-
-
-def command_config(args) -> dict[str, object] | None:
-    """Translate shared legacy add/schedule flags into runtime config overrides."""
-    if args.update or args.overwrite:
-        return {"ONLY_NEW": False}
-    return {"ONLY_NEW": bool(args.only_new)} if args.only_new is not None else None
-
-
 @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
 def cli_add(request: HttpRequest, args: AddCommandSchema):
     from archivebox.cli.archivebox_add import add
     from archivebox.misc.util import validate_url
 
+    config_overrides: dict[str, object] = {}
+    if args.only_new is not None:
+        config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
     submitted_urls: str | list[str] = args.urls
     if len(args.urls) == 1:
         try:
@@ -155,10 +144,20 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
             raise HttpError(400, "No valid URLs were submitted")
     crawl, snapshots = add(
         urls=submitted_urls,
-        **args.model_dump(exclude={"urls", "only_new", "update", "overwrite"}),
+        snapshot_ids=args.snapshot_ids,
+        tag=args.tag,
+        depth=args.depth,
+        max_urls=args.max_urls,
+        crawl_max_size=args.crawl_max_size,
+        crawl_timeout=args.crawl_timeout,
+        snapshot_max_size=args.snapshot_max_size,
+        index_only=args.index_only,
+        plugins=args.plugins,
+        persona=args.persona,
+        parser=args.parser,
         bg=True,  # Always run in background for API calls
         created_by_id=request.user.pk,
-        config=command_config(args),
+        config=config_overrides or None,
     )
 
     snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
@@ -168,7 +167,17 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         "snapshot_ids": snapshot_ids,
         "queued_urls": args.urls if isinstance(submitted_urls, str) else submitted_urls,
     }
-    return cli_response(request, result_payload, result_format="json")
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
+
+    return {
+        "success": True,
+        "errors": [],
+        "result": result_payload,
+        "result_format": "json",
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+    }
 
 
 @router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
@@ -195,20 +204,55 @@ def cli_update(request: HttpRequest, args: UpdateCommandSchema):
         ]
 
     update(**update_kwargs)
-    return cli_response(
-        request,
-        {"matched_count": len(matched_snapshot_ids), "snapshot_ids": matched_snapshot_ids} if is_filtered_update else None,
-        result_format="str",
-    )
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
+    return {
+        "success": True,
+        "errors": [],
+        "result": {
+            "matched_count": len(matched_snapshot_ids),
+            "snapshot_ids": matched_snapshot_ids,
+        }
+        if is_filtered_update
+        else None,
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+    }
 
 
 @router.post("/schedule", response=CLICommandResponseSchema, summary="archivebox schedule [args] [import_path]")
 def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
     from archivebox.cli.archivebox_schedule import schedule
 
-    result = schedule(**args.model_dump(exclude={"only_new", "update", "overwrite"}), config=command_config(args))
+    config_overrides: dict[str, object] = {}
+    if args.only_new is not None:
+        config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
+    result = schedule(
+        import_path=args.import_path,
+        add=args.add,
+        show=args.show,
+        foreground=args.foreground,
+        run_all=args.run_all,
+        quiet=args.quiet,
+        clear=args.clear,
+        every=args.every,
+        tag=args.tag,
+        depth=args.depth,
+        config=config_overrides or None,
+    )
 
-    return cli_response(request, result, result_format="json")
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "result_format": "json",
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+    }
 
 
 @router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
@@ -243,7 +287,16 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
     else:
         result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
 
-    return cli_response(request, result, result_format=result_format)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "result_format": result_format,
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+    }
 
 
 @router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
@@ -264,10 +317,13 @@ def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
         snapshots=snapshots_to_remove,
         timeout=timeout,
     )
-    return cli_response(
-        request,
-        result,
-        result_format="json",
-        success=bool(result["success"]),
-        errors=[str(result["error"])] if result["error"] else [],
-    )
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
+    return {
+        "success": bool(result["success"]),
+        "errors": [str(result["error"])] if result["error"] else [],
+        "result": result,
+        "result_format": "json",
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+    }

archivebox/api/v1_core/__init__.py

@@ -1,47 +0,0 @@
-"""Core REST routes, composed from entity-specific routers."""
-
-from typing import Union
-
-from django.db.models import Model
-from django.http import HttpRequest
-from django.shortcuts import redirect
-from ninja import Router
-from ninja.errors import HttpError
-
-from archivebox.api.v1_crawls import CrawlSchema, get_crawl
-
-from . import archiveresults, snapshots, tags
-from .archiveresults import get_archiveresult
-from .pagination import CustomPagination as CustomPagination
-from .schemas import ArchiveResultSchema, SnapshotSchema, TagSchema
-from .snapshots import get_snapshot
-from .tags import get_tag
-
-router = Router(tags=["Core Models"])
-router.add_router("", archiveresults.router)
-router.add_router("", snapshots.router)
-router.add_router("", tags.router)
-
-
-@router.get(
-    "/any/{id}",
-    response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema, CrawlSchema],
-    url_name="get_any",
-    summary="Get any object by its ID",
-)
-def get_any(request: HttpRequest, id: str):
-    """Get any object by its ID (e.g. snapshot, archiveresult, tag, crawl, etc.)."""
-    setattr(request, "with_snapshots", False)
-    setattr(request, "with_archiveresults", False)
-
-    for getter in (get_snapshot, get_archiveresult, get_tag, get_crawl):
-        try:
-            response = getter(request, id)
-            if isinstance(response, Model):
-                return redirect(
-                    f"/api/v1/{response._meta.app_label}/{response._meta.model_name}/{response.pk}?{request.META['QUERY_STRING']}",
-                )
-        except Exception:
-            pass
-
-    raise HttpError(404, "Object with given ID not found")

archivebox/api/v1_core/archiveresults.py

@@ -1,310 +0,0 @@
-import json
-import mimetypes
-import re
-from pathlib import Path, PurePosixPath
-from typing import Any
-
-from django.core.files.storage import FileSystemStorage
-from django.http import HttpRequest
-from django.http.multipartparser import MultiPartParser, MultiPartParserError
-from ninja import Form, Query, Router, UploadedFile
-from ninja.errors import HttpError
-from ninja.pagination import paginate
-
-from archivebox.core.models import ArchiveResult, Snapshot
-from archivebox.misc.db import uuid_ref_query as _uuid_ref_query
-
-from .lookup import _get_snapshot_by_ref
-from .pagination import CustomPagination
-from .schemas import ArchiveResultFilterSchema, ArchiveResultSchema
-
-router = Router(tags=["Core Models"])
-
-ARCHIVERESULT_UPLOAD_HOOK_NAME = Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME
-ARCHIVERESULT_UPLOAD_PLUGIN_RE = re.compile(r"^[A-Za-z0-9_.-]{1,32}$")
-
-
-@router.get("/archiveresults", response=list[ArchiveResultSchema], url_name="get_archiveresult")
-@paginate(CustomPagination)
-def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
-    """List all ArchiveResult entries matching these filters."""
-    queryset = filters.filter(ArchiveResult.objects.all())
-    if filters.search or filters.snapshot_tag:
-        return queryset.distinct()
-    return queryset
-
-
-@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
-def get_archiveresult(request: HttpRequest, archiveresult_id: str):
-    """Get a specific ArchiveResult by id."""
-    return ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
-
-
-def _normalize_uploaded_archiveresult_plugin(plugin: str) -> str:
-    normalized = str(plugin or "").strip().strip("/")
-    if not ARCHIVERESULT_UPLOAD_PLUGIN_RE.fullmatch(normalized):
-        raise HttpError(400, "Invalid ArchiveResult plugin name")
-    return normalized
-
-
-def _normalize_uploaded_archiveresult_output_path(output_path: str, *, filename: str) -> str:
-    raw_path = str(output_path or filename or "").strip().replace("\\", "/")
-    if not raw_path:
-        raise HttpError(400, "ArchiveResult output path is required")
-
-    path = PurePosixPath(raw_path)
-    if path.is_absolute() or any(part in ("", ".", "..") for part in path.parts):
-        raise HttpError(400, "Invalid ArchiveResult output path")
-
-    return str(path)
-
-
-def _parse_archiveresult_output_json(output_json: str | None) -> dict[str, Any] | None:
-    if not output_json:
-        return None
-    try:
-        parsed = json.loads(output_json)
-    except json.JSONDecodeError as err:
-        raise HttpError(400, "ArchiveResult output_json must be valid JSON") from err
-    if parsed is None:
-        return None
-    if not isinstance(parsed, dict):
-        raise HttpError(400, "ArchiveResult output_json must be a JSON object")
-    return parsed
-
-
-def _get_archiveresult_upload_data(request: HttpRequest):
-    cached = request.__dict__.get("_archiveresult_upload_data")
-    if cached is not None:
-        return cached
-
-    if request.method.upper() == "PATCH" and request.content_type.startswith("multipart/"):
-        try:
-            data = MultiPartParser(request.META, request, request.upload_handlers, request.encoding).parse()
-        except MultiPartParserError as err:
-            raise HttpError(400, f"Invalid ArchiveResult multipart upload: {err}") from err
-    else:
-        data = (request.POST, request.FILES)
-
-    setattr(request, "_archiveresult_upload_data", data)
-    return data
-
-
-def _get_archiveresult_upload_files(request: HttpRequest, *, allow_empty: bool = False) -> list[UploadedFile]:
-    _post, request_files = _get_archiveresult_upload_data(request)
-    files = [*request_files.getlist("files"), *request_files.getlist("file")]
-    if not files and not allow_empty:
-        raise HttpError(400, "At least one ArchiveResult file is required")
-    return files
-
-
-def _get_archiveresult_upload_form_values(request: HttpRequest, *field_names: str) -> list[str]:
-    request_post, _files = _get_archiveresult_upload_data(request)
-    values: list[str] = []
-    for field_name in field_names:
-        values.extend(str(value) for value in request_post.getlist(field_name))
-    if len(values) == 1:
-        value = values[0].strip()
-        if value.startswith("["):
-            try:
-                parsed = json.loads(value)
-            except json.JSONDecodeError:
-                parsed = None
-            if isinstance(parsed, list):
-                return [str(item) for item in parsed]
-    return values
-
-
-def _get_archiveresult_upload_form_value(request: HttpRequest, *field_names: str) -> str:
-    request_post, _files = _get_archiveresult_upload_data(request)
-    for field_name in field_names:
-        value = request_post.get(field_name)
-        if value is not None:
-            return str(value)
-    return ""
-
-
-def _parse_archiveresult_upload_int(value: str, field_name: str, *, default: int | None = None) -> int:
-    if value == "" and default is not None:
-        return default
-    try:
-        parsed = int(value)
-    except (TypeError, ValueError) as err:
-        raise HttpError(400, f"ArchiveResult {field_name} must be an integer") from err
-    if parsed < 0:
-        raise HttpError(400, f"ArchiveResult {field_name} must be non-negative")
-    return parsed
-
-
-def _write_archiveresult_files(
-    request: HttpRequest,
-    snapshot: Snapshot,
-    plugin_name: str,
-    *,
-    existing_output_files: dict[str, dict[str, Any]] | None = None,
-    allow_empty: bool = False,
-) -> dict[str, dict[str, Any]]:
-    files = _get_archiveresult_upload_files(request, allow_empty=allow_empty)
-    output_paths = _get_archiveresult_upload_form_values(request, "output_paths", "output_path")
-    mime_types = _get_archiveresult_upload_form_values(request, "mime_types", "mime_type")
-    chunk_output_path = _get_archiveresult_upload_form_value(request, "chunk_output_path")
-
-    snapshot_dir = snapshot.output_dir
-    plugin_dir = snapshot_dir / plugin_name
-    storage = FileSystemStorage(location=str(plugin_dir))
-    output_files = dict(existing_output_files or {})
-
-    if not files:
-        return output_files
-
-    if chunk_output_path:
-        if len(files) != 1:
-            raise HttpError(400, "Exactly one ArchiveResult file chunk is required")
-
-        uploaded_file = files[0]
-        relative_output_path = _normalize_uploaded_archiveresult_output_path(
-            chunk_output_path,
-            filename=uploaded_file.name,
-        )
-        chunk_index = _parse_archiveresult_upload_int(
-            _get_archiveresult_upload_form_value(request, "chunk_index"),
-            "chunk_index",
-        )
-        chunk_count = _parse_archiveresult_upload_int(
-            _get_archiveresult_upload_form_value(request, "chunk_count"),
-            "chunk_count",
-        )
-        chunk_offset = _parse_archiveresult_upload_int(
-            _get_archiveresult_upload_form_value(request, "chunk_offset"),
-            "chunk_offset",
-        )
-        chunk_total_size = _parse_archiveresult_upload_int(
-            _get_archiveresult_upload_form_value(request, "chunk_total_size"),
-            "chunk_total_size",
-        )
-
-        if chunk_count < 1:
-            raise HttpError(400, "ArchiveResult chunk_count must be at least 1")
-        if chunk_index >= chunk_count:
-            raise HttpError(400, "ArchiveResult chunk_index must be less than chunk_count")
-        if chunk_total_size and chunk_offset > chunk_total_size:
-            raise HttpError(400, "ArchiveResult chunk_offset cannot exceed chunk_total_size")
-
-        if chunk_index == 0 and chunk_offset == 0 and storage.exists(relative_output_path):
-            storage.delete(relative_output_path)
-
-        current_size = storage.size(relative_output_path) if storage.exists(relative_output_path) else 0
-        if current_size != chunk_offset:
-            raise HttpError(
-                409,
-                f"ArchiveResult chunk offset mismatch for {relative_output_path}: expected {current_size}, got {chunk_offset}",
-            )
-
-        Path(storage.path(relative_output_path)).parent.mkdir(parents=True, exist_ok=True)
-        with storage.open(relative_output_path, "ab") as destination:
-            for chunk in uploaded_file.chunks():
-                destination.write(chunk)
-
-        size = storage.size(relative_output_path)
-        upload_complete = chunk_index + 1 == chunk_count
-        if upload_complete and size != chunk_total_size:
-            raise HttpError(
-                409,
-                f"ArchiveResult chunk size mismatch for {relative_output_path}: expected {chunk_total_size}, got {size}",
-            )
-
-        guessed_mime = mimetypes.guess_type(relative_output_path)[0]
-        output_mime_type = (mime_types[0] if mime_types else "") or uploaded_file.content_type or guessed_mime or "application/octet-stream"
-        output_files[relative_output_path] = {
-            "extension": PurePosixPath(relative_output_path).suffix.lower().lstrip("."),
-            "mimetype": output_mime_type,
-            "size": size,
-            "upload": {
-                "chunked": True,
-                "chunk_index": chunk_index,
-                "chunk_count": chunk_count,
-                "chunks_received": chunk_index + 1,
-                "complete": upload_complete,
-            },
-        }
-
-        return output_files
-
-    for index, uploaded_file in enumerate(files):
-        relative_output_path = _normalize_uploaded_archiveresult_output_path(
-            output_paths[index] if index < len(output_paths) else "",
-            filename=uploaded_file.name,
-        )
-        if storage.exists(relative_output_path):
-            storage.delete(relative_output_path)
-        saved_output_path = storage.save(relative_output_path, uploaded_file)
-        size = storage.size(saved_output_path)
-        guessed_mime = mimetypes.guess_type(saved_output_path)[0]
-        output_mime_type = (
-            (mime_types[index] if index < len(mime_types) else "")
-            or uploaded_file.content_type
-            or guessed_mime
-            or "application/octet-stream"
-        )
-        output_files[saved_output_path] = {
-            "extension": PurePosixPath(saved_output_path).suffix.lower().lstrip("."),
-            "mimetype": output_mime_type,
-            "size": size,
-        }
-
-    return output_files
-
-
-@router.post(
-    "/archiveresults",
-    response=ArchiveResultSchema,
-    url_name="create_archiveresult",
-)
-def create_archiveresult(
-    request: HttpRequest,
-    snapshot_id: str = Form(...),
-    plugin: str = Form(...),
-    output_str: str = Form(""),
-    hook_name: str = Form(ARCHIVERESULT_UPLOAD_HOOK_NAME),
-    status: str = Form(str(ArchiveResult.StatusChoices.SUCCEEDED)),
-    output_json: str = Form(""),
-):
-    """Create or update an ArchiveResult with one or more output files."""
-    snapshot = _get_snapshot_by_ref(snapshot_id)
-    plugin_name = _normalize_uploaded_archiveresult_plugin(plugin)
-    result = ArchiveResult.objects.filter(
-        snapshot=snapshot,
-        plugin=plugin_name,
-        hook_name=hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME,
-    ).first()
-    result = result or ArchiveResult(snapshot=snapshot, plugin=plugin_name, hook_name=hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME)
-    metadata = {"output_str": output_str, "status": status, "output_json": _parse_archiveresult_output_json(output_json)}
-    files = _write_archiveresult_files(request, snapshot, plugin_name, allow_empty=True)
-    try:
-        return result.apply_upload(files, metadata=metadata, replace_metadata=True)
-    except ArchiveResult.UploadConflict as err:
-        raise HttpError(409, str(err)) from err
-
-
-@router.patch("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="patch_archiveresult")
-def patch_archiveresult(
-    request: HttpRequest,
-    archiveresult_id: str,
-):
-    """Append or replace files on an existing ArchiveResult."""
-    result = ArchiveResult.objects.select_related("snapshot__crawl__created_by").get(_uuid_ref_query("id", archiveresult_id))
-    uploaded_output_files = _write_archiveresult_files(
-        request,
-        result.snapshot,
-        result.plugin,
-    )
-    output_str = _get_archiveresult_upload_form_value(request, "output_str")
-    status = _get_archiveresult_upload_form_value(request, "status")
-    output_json = _get_archiveresult_upload_form_value(request, "output_json")
-    metadata = {key: value for key, value in {"output_str": output_str, "status": status}.items() if value}
-    if output_json:
-        metadata["output_json"] = _parse_archiveresult_output_json(output_json)
-    try:
-        return result.apply_upload(uploaded_output_files, metadata=metadata)
-    except ArchiveResult.UploadConflict as err:
-        raise HttpError(409, str(err)) from err

archivebox/api/v1_core/lookup.py

@@ -1,12 +0,0 @@
-from django.db.models import Q
-
-from archivebox.core.models import Snapshot
-from archivebox.misc.db import uuid_ref_query as _uuid_ref_query
-
-
-def _get_snapshot_by_ref(snapshot_id: str):
-    queryset = Snapshot.objects.select_related("crawl__created_by")
-    try:
-        return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        return queryset.get(_uuid_ref_query("id", snapshot_id))

archivebox/api/v1_core/pagination.py

@@ -1,40 +0,0 @@
-import math
-from typing import Any
-
-from django.http import HttpRequest
-from ninja.pagination import PaginationBase
-
-
-class CustomPagination(PaginationBase):
-    class Input(PaginationBase.Input):
-        limit: int = 200
-        offset: int = 0
-        page: int = 0
-
-    class Output(PaginationBase.Output):
-        count: int
-        total_items: int
-        total_pages: int
-        page: int
-        limit: int
-        offset: int
-        num_items: int
-        items: list[Any]
-
-    def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
-        limit = min(pagination.limit, 500)
-        offset = pagination.offset or (pagination.page * limit)
-        total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count()
-        total_pages = math.ceil(total / limit)
-        current_page = math.ceil(offset / (limit + 1))
-        items = queryset[offset : offset + limit]
-        return {
-            "count": total,
-            "total_items": total,
-            "total_pages": total_pages,
-            "page": current_page,
-            "limit": limit,
-            "offset": offset,
-            "num_items": len(items),
-            "items": items,
-        }

archivebox/api/v1_core/schemas.py

@@ -1,301 +0,0 @@
-from collections import defaultdict
-from datetime import datetime
-from typing import Annotated, Any
-from uuid import UUID
-
-from django.db.models import Q
-from archivebox.api.schemas import OwnedObjectSchema
-
-from ninja import FilterLookup, FilterSchema, Schema
-
-from archivebox.core.models import ArchiveResult, Snapshot
-
-
-class MinimalArchiveResultSchema(OwnedObjectSchema):
-    TYPE: str = "core.models.ArchiveResult"
-    id: UUID
-    created_at: datetime | None
-    modified_at: datetime | None
-    status: str
-    retry_at: datetime | None = None
-    plugin: str
-    hook_name: str
-    process_id: UUID | None
-    cmd_version: str | None
-    cmd: list[str] | None
-    pwd: str | None
-    output_str: str
-    output_json: dict[str, Any] | None
-    output_files: dict[str, dict[str, Any]] | None
-    output_size: int
-    output_mimetypes: str
-    start_ts: datetime | None
-    end_ts: datetime | None
-
-    @staticmethod
-    def resolve_output_files(obj):
-        return obj.output_file_map()
-
-    @staticmethod
-    def resolve_output_mimetypes(obj) -> str:
-        mime_sizes: dict[str, int] = defaultdict(int)
-        for metadata in obj.output_file_map().values():
-            if not isinstance(metadata, dict):
-                continue
-            mimetype = str(metadata.get("mimetype") or "").strip()
-            try:
-                size = max(int(metadata.get("size") or 0), 0)
-            except (TypeError, ValueError):
-                size = 0
-            if mimetype and size:
-                mime_sizes[mimetype] += size
-        if mime_sizes:
-            return ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
-        return obj.output_mimetypes or ""
-
-
-class ArchiveResultSchema(MinimalArchiveResultSchema):
-    TYPE: str = "core.models.ArchiveResult"
-    snapshot_id: UUID
-    snapshot_timestamp: str
-    snapshot_url: str
-    snapshot_tags: list[str]
-
-    @staticmethod
-    def resolve_snapshot_timestamp(obj):
-        return obj.snapshot.timestamp
-
-    @staticmethod
-    def resolve_snapshot_url(obj):
-        return obj.snapshot.url
-
-    @staticmethod
-    def resolve_snapshot_id(obj):
-        return obj.snapshot_id
-
-    @staticmethod
-    def resolve_snapshot_tags(obj):
-        return sorted(tag.name for tag in obj.snapshot.tags.all())
-
-
-class ArchiveResultFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup(["id__startswith", "snapshot__id__startswith", "snapshot__timestamp__startswith"])] = None
-    search: Annotated[
-        str | None,
-        FilterLookup(
-            [
-                "snapshot__url__icontains",
-                "snapshot__title__icontains",
-                "snapshot__tags__name__icontains",
-                "plugin",
-                "output_str__icontains",
-                "id__startswith",
-                "snapshot__id__startswith",
-                "snapshot__timestamp__startswith",
-            ],
-        ),
-    ] = None
-    snapshot_id: Annotated[str | None, FilterLookup(["snapshot__id__startswith", "snapshot__timestamp__startswith"])] = None
-    snapshot_url: Annotated[str | None, FilterLookup("snapshot__url__icontains")] = None
-    snapshot_tag: Annotated[str | None, FilterLookup("snapshot__tags__name__icontains")] = None
-    status: Annotated[str | None, FilterLookup("status")] = None
-    output_str: Annotated[str | None, FilterLookup("output_str__icontains")] = None
-    plugin: Annotated[str | None, FilterLookup("plugin__icontains")] = None
-    hook_name: Annotated[str | None, FilterLookup("hook_name__icontains")] = None
-    process_id: Annotated[str | None, FilterLookup("process__id__startswith")] = None
-    cmd: Annotated[str | None, FilterLookup("cmd__0__icontains")] = None
-    pwd: Annotated[str | None, FilterLookup("pwd__icontains")] = None
-    cmd_version: Annotated[str | None, FilterLookup("cmd_version")] = None
-    created_at: Annotated[datetime | None, FilterLookup("created_at")] = None
-    created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
-    created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
-
-
-class SnapshotSchema(OwnedObjectSchema):
-    TYPE: str = "core.models.Snapshot"
-    id: UUID
-    created_at: datetime
-    modified_at: datetime
-    status: str
-    retry_at: datetime | None
-    bookmarked_at: datetime
-    downloaded_at: datetime | None
-    url: str
-    tags: list[str]
-    title: str | None
-    timestamp: str
-    archive_path: str
-    archive_size: int
-    output_size: int
-    num_archiveresults: int
-    archiveresults: list[MinimalArchiveResultSchema]
-
-    @staticmethod
-    def resolve_tags(obj):
-        return sorted(tag.name for tag in obj.tags.all())
-
-    @staticmethod
-    def resolve_archive_size(obj):
-        return int(obj.archive_size or 0)
-
-    @staticmethod
-    def resolve_output_size(obj):
-        return SnapshotSchema.resolve_archive_size(obj)
-
-    @staticmethod
-    def resolve_num_archiveresults(obj, context):
-        return obj.archiveresult_set.all().distinct().count()
-
-    @staticmethod
-    def resolve_archiveresults(obj, context):
-        if bool(context["request"].__dict__.get("with_archiveresults", False)):
-            return obj.archiveresult_set.all().distinct()
-        return ArchiveResult.objects.none()
-
-
-class SnapshotUpdateSchema(Schema):
-    action: str | None = None
-    status: str | None = None
-    retry_at: datetime | None = None
-    tags: list[str] | None = None
-
-
-class SnapshotCreateSchema(Schema):
-    url: str
-    crawl_id: str | None = None
-    depth: int = 0
-    title: str | None = None
-    tags: list[str] | None = None
-    status: str | None = None
-
-
-class SnapshotDeleteResponseSchema(Schema):
-    success: bool
-    snapshot_id: str
-    crawl_id: str
-    deleted_count: int
-
-
-class SnapshotFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup(["id__istartswith", "id__iendswith", "timestamp__startswith"])] = None
-    created_by_id: Annotated[str | None, FilterLookup("crawl__created_by_id")] = None
-    created_by_username: Annotated[str | None, FilterLookup("crawl__created_by__username__icontains")] = None
-    created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
-    created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
-    created_at: Annotated[datetime | None, FilterLookup("created_at")] = None
-    modified_at: Annotated[datetime | None, FilterLookup("modified_at")] = None
-    modified_at__gte: Annotated[datetime | None, FilterLookup("modified_at__gte")] = None
-    modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
-    search: str | None = None
-    search_mode: str | None = None
-    status: str | None = None
-    url: Annotated[str | None, FilterLookup("url")] = None
-    tag: Annotated[str | None, FilterLookup("tags__name")] = None
-    title: Annotated[str | None, FilterLookup("title__icontains")] = None
-    timestamp: Annotated[str | None, FilterLookup("timestamp__startswith")] = None
-    bookmarked_at__gte: Annotated[datetime | None, FilterLookup("bookmarked_at__gte")] = None
-    bookmarked_at__lt: Annotated[datetime | None, FilterLookup("bookmarked_at__lt")] = None
-
-    def filter_search(self, value: str | None) -> Q:
-        return Q()
-
-    def filter_search_mode(self, value: str | None) -> Q:
-        return Q()
-
-    def filter_status(self, value: str | None) -> Q:
-        return Q()
-
-
-class TagSchema(OwnedObjectSchema):
-    TYPE: str = "core.models.Tag"
-    id: int
-    modified_at: datetime
-    created_at: datetime
-    name: str
-    num_snapshots: int
-    snapshots: list[SnapshotSchema]
-
-    @staticmethod
-    def resolve_num_snapshots(obj, context):
-        return obj.snapshot_set.all().distinct().count()
-
-    @staticmethod
-    def resolve_snapshots(obj, context):
-        if bool(context["request"].__dict__.get("with_snapshots", False)):
-            return obj.snapshot_set.all().distinct()
-        return Snapshot.objects.none()
-
-
-class TagAutocompleteSchema(Schema):
-    tags: list[dict]
-
-
-class TagCreateSchema(Schema):
-    name: str
-
-
-class TagCreateResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    tag_name: str
-    created: bool
-
-
-class TagSearchSnapshotSchema(Schema):
-    id: str
-    title: str
-    url: str
-    favicon_url: str
-    admin_url: str
-    archive_url: str
-    downloaded_at: str | None = None
-
-
-class TagSearchCardSchema(Schema):
-    id: int
-    name: str
-    slug: str
-    num_snapshots: int
-    filter_url: str
-    edit_url: str
-    export_urls_url: str
-    export_jsonl_url: str
-    rename_url: str
-    delete_url: str
-    snapshots: list[TagSearchSnapshotSchema]
-
-
-class TagSearchResponseSchema(Schema):
-    tags: list[TagSearchCardSchema]
-    sort: str
-    created_by: str
-    year: str
-    has_snapshots: str
-
-
-class TagUpdateSchema(Schema):
-    name: str
-
-
-class TagUpdateResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    tag_name: str
-
-
-class TagDeleteResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    deleted_count: int
-
-
-class TagSnapshotRequestSchema(Schema):
-    snapshot_id: str
-    tag_name: str | None = None
-    tag_id: int | None = None
-
-
-class TagSnapshotResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    tag_name: str

archivebox/api/v1_core/snapshots.py

@@ -1,356 +0,0 @@
-from datetime import datetime, time
-from html import unescape
-from typing import Any
-
-from django.contrib.auth import get_user_model
-from django.contrib.auth.models import User
-from django.core.exceptions import ValidationError
-from django.db import IntegrityError
-from django.db.models import Q
-from django.http import HttpRequest, HttpResponse
-from django.utils import timezone
-from django.utils.dateparse import parse_date, parse_datetime
-from django.utils.feedgenerator import Rss201rev2Feed
-from ninja import Query, Router
-from ninja.errors import HttpError
-from ninja.pagination import paginate
-
-from archivebox.api.v1_crawls import get_crawl_by_ref
-from archivebox.core.models import Snapshot
-from archivebox.core.routes_util import build_web_url
-from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status
-from archivebox.crawls.locks import crawl_lifecycle_lock
-from archivebox.crawls.models import Crawl
-from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
-from archivebox.search.config import get_search_mode, get_search_mode_backend
-from archivebox.search.query import apply_snapshot_search
-
-from .lookup import _get_snapshot_by_ref
-from .pagination import CustomPagination
-from .schemas import SnapshotCreateSchema, SnapshotDeleteResponseSchema, SnapshotFilterSchema, SnapshotSchema, SnapshotUpdateSchema
-
-router = Router(tags=["Core Models"])
-
-
-def normalize_tag_list(tags: list[str] | None = None) -> list[str]:
-    return [tag.strip() for tag in (tags or []) if tag and tag.strip()]
-
-
-def _parse_rss_before(before: str | None) -> datetime:
-    if not before:
-        return timezone.now()
-
-    value = before.strip()
-    parsed_dt = None
-
-    if len(value) == 8 and value.isdigit():
-        parsed_date = datetime.strptime(value, "%Y%m%d").date()
-    else:
-        parsed_dt = parse_datetime(value)
-        parsed_date = None if parsed_dt else parse_date(value)
-
-    if parsed_dt is None:
-        if parsed_date is None:
-            raise HttpError(400, "before must be an ISO datetime, YYYY-MM-DD, or YYYYMMDD")
-        parsed_dt = datetime.combine(parsed_date, time.max)
-
-    if timezone.is_naive(parsed_dt):
-        parsed_dt = timezone.make_aware(parsed_dt, timezone.get_current_timezone())
-    return parsed_dt
-
-
-def _filter_snapshots_for_rss(
-    *,
-    crawl_id: str = "",
-    created_by: str = "",
-    before: str | None = None,
-    limit: int = 50,
-):
-    limit = max(1, min(int(limit or 50), 500))
-    before_dt = _parse_rss_before(before)
-    queryset = (
-        Snapshot.objects.select_related("crawl__created_by")
-        .prefetch_related("tags")
-        .only(
-            "id",
-            "url",
-            "title",
-            "timestamp",
-            "bookmarked_at",
-            "created_at",
-            "modified_at",
-            "fs_version",
-            "crawl_id",
-            "crawl__id",
-            "crawl__created_by_id",
-            "crawl__created_by__id",
-            "crawl__created_by__username",
-        )
-        .filter(bookmarked_at__lte=before_dt)
-    )
-    crawl_id = crawl_id.strip()
-    if crawl_id:
-        matching_crawl_pks = list(filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).values_list("pk", flat=True)[:100])
-        queryset = queryset.filter(crawl_id__in=matching_crawl_pks)
-
-    created_by = created_by.strip()
-    if created_by:
-        created_by_query = Q(crawl__created_by__username__iexact=created_by)
-        user_model = get_user_model()
-        try:
-            prepared_pk = user_model._meta.pk.get_prep_value(created_by)
-        except (TypeError, ValueError, ValidationError):
-            prepared_pk = None
-        if prepared_pk not in (None, ""):
-            created_by_query |= Q(crawl__created_by_id=prepared_pk)
-        queryset = queryset.filter(created_by_query)
-
-    return queryset.order_by("-bookmarked_at", "-created_at", "-id")[:limit]
-
-
-def _snapshots_rss_response(
-    request: HttpRequest,
-    *,
-    snapshots,
-    title: str = "ArchiveBox Snapshots",
-) -> HttpResponse:
-    web_base_url = build_web_url("/", request=request).rstrip("/")
-    feed_query = request.GET.copy()
-    for sensitive_param in ("api_key", "token", "password"):
-        feed_query.pop(sensitive_param, None)
-    feed_path = request.path
-    feed_url = request.build_absolute_uri(f"{feed_path}?{feed_query.urlencode()}" if feed_query else feed_path)
-
-    feed = Rss201rev2Feed(
-        title=title,
-        link=build_web_url("/public/", request=request),
-        description="Recently added ArchiveBox snapshots.",
-        language="en",
-        feed_url=feed_url,
-    )
-
-    for snapshot in snapshots:
-        archived_url = build_web_url(f"/{snapshot.archive_path_from_db}", request=request)
-        tags = [tag.name for tag in snapshot.tags.all()]
-        crawl_user = snapshot.crawl.created_by if snapshot.crawl_id else None
-        description = f"Original URL: {snapshot.url}\nArchived snapshot: {archived_url}"
-        feed.add_item(
-            title=unescape(snapshot.title or snapshot.url),
-            link=archived_url or web_base_url,
-            description=description,
-            unique_id=str(snapshot.id),
-            unique_id_is_permalink=False,
-            pubdate=snapshot.bookmarked_at or snapshot.created_at,
-            updateddate=snapshot.modified_at,
-            author_name=crawl_user.username if crawl_user else None,
-            categories=tags,
-        )
-
-    return HttpResponse(feed.writeString("utf-8"), content_type="application/rss+xml; charset=utf-8")
-
-
-@router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
-@paginate(CustomPagination)
-def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
-    """List all Snapshot entries matching these filters."""
-    setattr(request, "with_archiveresults", with_archiveresults)
-    try:
-        queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-    queryset = filters.filter(queryset).distinct()
-    query = (filters.search or "").strip()
-    if not query:
-        return queryset
-
-    runtime_config = request.archivebox_config
-    search_mode = get_search_mode(filters.search_mode, config=runtime_config)
-    try:
-        return apply_snapshot_search(
-            queryset,
-            query,
-            search_mode=search_mode,
-            config=runtime_config,
-            include_id_matches=True,
-        )
-    except Exception:
-        if get_search_mode_backend(search_mode, config=runtime_config):
-            return queryset.none()
-        return apply_snapshot_search(queryset, query, search_mode="meta", config=runtime_config, include_id_matches=True)
-
-
-@router.get("/snapshots.rss", url_name="get_snapshots_rss")
-def get_snapshots_rss(
-    request: HttpRequest,
-    crawl_id: str = "",
-    created_by: str = "",
-    limit: int = 50,
-    before: str | None = None,
-):
-    """Return matching snapshots as an RSS feed, newest first."""
-    snapshots = _filter_snapshots_for_rss(
-        crawl_id=crawl_id,
-        created_by=created_by,
-        limit=limit,
-        before=before,
-    )
-    return _snapshots_rss_response(request, snapshots=snapshots)
-
-
-@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
-def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
-    """Get a specific Snapshot by id."""
-    setattr(request, "with_archiveresults", with_archiveresults)
-    return _get_snapshot_by_ref(snapshot_id)
-
-
-@router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
-def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
-    tags = normalize_tag_list(data.tags)
-    try:
-        status = normalize_snapshot_status(data.status)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-    if not data.url.strip():
-        raise HttpError(400, "URL is required")
-    try:
-        validate_url_length(data.url.strip())
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-    if data.depth not in (0, 1, 2, 3, 4):
-        raise HttpError(400, "depth must be between 0 and 4")
-
-    if data.crawl_id:
-        crawl = get_crawl_by_ref(data.crawl_id)
-        crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
-        tags = tags or crawl_tags
-    else:
-        crawl = Crawl.objects.create(
-            urls=data.url,
-            max_depth=max(data.depth, 0),
-            tags_str=",".join(tags),
-            status=Crawl.StatusChoices.QUEUED,
-            retry_at=timezone.now(),
-            created_by=request.user if isinstance(request.user, User) else None,
-        )
-
-    # Browser uploads must not wait behind the runner's crawl-wide lifecycle
-    # lock. The unique insert recovery and CAS update below are the request-side
-    # coordination boundary for this idempotent metadata sync.
-    snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
-    if snapshot is None:
-        try:
-            snapshot = Snapshot.objects.create(
-                url=data.url,
-                crawl=crawl,
-                depth=data.depth,
-                title=data.title,
-                timestamp=str(timezone.now().timestamp()),
-                status=status or Snapshot.StatusChoices.QUEUED,
-                retry_at=timezone.now(),
-            )
-        except IntegrityError:
-            snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
-            if snapshot is None:
-                raise
-
-    for _attempt in range(3):
-        updates: dict[str, Any] = {}
-        if data.title is not None and snapshot.title != data.title:
-            updates["title"] = data.title
-        if status is not None and snapshot.status != status:
-            updates["status"] = status
-        if not updates or snapshot.safe_update(updates, extra_filter={"modified_at": snapshot.modified_at}):
-            break
-    else:
-        raise HttpError(409, "Snapshot changed while metadata was being updated")
-
-    if tags:
-        snapshot.save_tags(
-            tags,
-            created_by=request.user if isinstance(request.user, User) else None,
-        )
-
-    try:
-        snapshot.ensure_crawl_symlink()
-    except Exception:
-        pass
-
-    setattr(request, "with_archiveresults", False)
-    return snapshot
-
-
-@router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
-def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
-    """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
-    snapshot = _get_snapshot_by_ref(snapshot_id)
-    crawl_id = str(snapshot.crawl_id)
-
-    payload = data.dict(exclude_unset=True)
-    update_fields = ["modified_at"]
-    action = payload.pop("action", None)
-    tags = payload.pop("tags", None)
-
-    if action:
-        if action == "pause":
-            snapshot.pause()
-            setattr(request, "with_archiveresults", False)
-            return snapshot
-        if action in ("resume", "unpause"):
-            snapshot.resume()
-            setattr(request, "with_archiveresults", False)
-            return snapshot
-        if action == "cancel":
-            snapshot.cancel()
-            setattr(request, "with_archiveresults", False)
-            return snapshot
-        raise HttpError(400, f"Invalid action: {action}")
-
-    with crawl_lifecycle_lock(crawl_id):
-        snapshot = _get_snapshot_by_ref(snapshot_id)
-        if "status" in payload:
-            try:
-                snapshot.status = normalize_snapshot_status(payload["status"])
-            except ValueError as err:
-                raise HttpError(400, str(err)) from err
-            if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
-                snapshot.retry_at = None
-            update_fields.append("status")
-
-        if "retry_at" in payload:
-            snapshot.retry_at = payload["retry_at"]
-            update_fields.append("retry_at")
-
-        if tags is not None:
-            snapshot.save_tags(
-                normalize_tag_list(tags),
-                created_by=request.user if isinstance(request.user, User) else None,
-            )
-
-        if payload.get("status") == Snapshot.StatusChoices.SEALED:
-            snapshot.cancel()
-        else:
-            snapshot.save(update_fields=update_fields)
-    setattr(request, "with_archiveresults", False)
-    return snapshot
-
-
-@router.delete("/snapshot/{snapshot_id}", response=SnapshotDeleteResponseSchema, url_name="delete_snapshot")
-def delete_snapshot(request: HttpRequest, snapshot_id: str):
-    snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
-    snapshot_id_str = str(snapshot.id)
-    crawl_id_str = str(snapshot.crawl.pk)
-    snapshot.cancel()
-
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        snapshot = get_snapshot(request, snapshot_id_str, with_archiveresults=False)
-        deleted_count, _ = snapshot.delete()
-    return {
-        "success": True,
-        "snapshot_id": snapshot_id_str,
-        "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
-    }

archivebox/api/v1_core/tags.py

@@ -1,285 +0,0 @@
-from django.core.exceptions import ValidationError
-from django.db.models import Q
-from django.http import HttpRequest, HttpResponse
-from ninja import Router
-from ninja.errors import HttpError
-from ninja.pagination import paginate
-
-from archivebox.api.auth import authenticated_user_from_request
-from archivebox.config.common import get_config
-from archivebox.core.models import Snapshot, Tag
-from archivebox.core.permissions import public_snapshots_queryset
-from archivebox.core.tag_util import (
-    add_snapshot_counts,
-    build_tag_cards,
-    export_tag_snapshots_jsonl,
-    export_tag_urls,
-    get_matching_tags,
-    get_tag_by_ref,
-    normalize_created_by_filter,
-    normalize_created_year_filter,
-    normalize_has_snapshots_filter,
-    normalize_tag_sort,
-)
-
-from .pagination import CustomPagination
-from .schemas import (
-    TagAutocompleteSchema,
-    TagCreateResponseSchema,
-    TagCreateSchema,
-    TagDeleteResponseSchema,
-    TagSchema,
-    TagSearchResponseSchema,
-    TagSnapshotRequestSchema,
-    TagSnapshotResponseSchema,
-    TagUpdateResponseSchema,
-    TagUpdateSchema,
-)
-
-router = Router(tags=["Core Models"])
-
-
-@router.get("/tags", response=list[TagSchema], url_name="get_tags")
-@paginate(CustomPagination)
-def get_tags(request: HttpRequest):
-    setattr(request, "with_snapshots", False)
-    setattr(request, "with_archiveresults", False)
-    return get_matching_tags()
-
-
-@router.get("/tag/{tag_id}", response=TagSchema, url_name="get_tag")
-def get_tag(request: HttpRequest, tag_id: str, with_snapshots: bool = True):
-    setattr(request, "with_snapshots", with_snapshots)
-    setattr(request, "with_archiveresults", False)
-    try:
-        return get_tag_by_ref(tag_id)
-    except (Tag.DoesNotExist, ValidationError):
-        raise HttpError(404, "Tag not found")
-
-
-def _get_snapshot_for_tag_edit(snapshot_ref: str) -> Snapshot:
-    snapshot_ref = str(snapshot_ref or "").strip().lower()
-    if not snapshot_ref:
-        raise HttpError(400, "Snapshot id is required")
-
-    snapshot_qs = Snapshot.objects.only("id")
-    is_full_uuid = len(snapshot_ref.replace("-", "")) == 32 and all(char in "0123456789abcdef-" for char in snapshot_ref)
-    if is_full_uuid:
-        try:
-            return snapshot_qs.get(pk=snapshot_ref.replace("-", ""))
-        except (Snapshot.DoesNotExist, ValueError):
-            pass
-
-    if len(snapshot_ref) >= 14:
-        try:
-            return snapshot_qs.get(timestamp=snapshot_ref)
-        except Snapshot.DoesNotExist:
-            pass
-        except Snapshot.MultipleObjectsReturned:
-            snapshot = snapshot_qs.filter(timestamp=snapshot_ref).first()
-            if snapshot is not None:
-                return snapshot
-
-    try:
-        return snapshot_qs.get(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref))
-    except Snapshot.DoesNotExist:
-        raise HttpError(404, "Snapshot not found") from None
-    except Snapshot.MultipleObjectsReturned:
-        snapshot = snapshot_qs.filter(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref)).first()
-        if snapshot is None:
-            raise HttpError(404, "Snapshot not found")
-        return snapshot
-
-
-@router.get("/tags/search/", response=TagSearchResponseSchema, url_name="search_tags")
-def search_tags(
-    request: HttpRequest,
-    q: str = "",
-    sort: str = "created_desc",
-    created_by: str = "",
-    year: str = "",
-    has_snapshots: str = "all",
-):
-    """Return detailed tag cards for admin/live-search UIs."""
-    normalized_sort = normalize_tag_sort(sort)
-    normalized_created_by = normalize_created_by_filter(created_by)
-    normalized_year = normalize_created_year_filter(year)
-    normalized_has_snapshots = normalize_has_snapshots_filter(has_snapshots)
-    return {
-        "tags": build_tag_cards(
-            query=q,
-            request=request,
-            preview_limit=0,
-            sort=normalized_sort,
-            created_by=normalized_created_by,
-            year=normalized_year,
-            has_snapshots=normalized_has_snapshots,
-        ),
-        "sort": normalized_sort,
-        "created_by": normalized_created_by,
-        "year": normalized_year,
-        "has_snapshots": normalized_has_snapshots,
-    }
-
-
-def _public_tag_listing_enabled() -> bool:
-    return get_config().PUBLIC_INDEX
-
-
-def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
-    if authenticated_user_from_request(request):
-        return True
-
-    return _public_tag_listing_enabled()
-
-
-@router.get("/tags/autocomplete/", response=TagAutocompleteSchema, url_name="tags_autocomplete", auth=None)
-def tags_autocomplete(request: HttpRequest, q: str = ""):
-    """Return tags matching the query for autocomplete."""
-    if not _request_has_tag_autocomplete_access(request):
-        raise HttpError(401, "Authentication required")
-
-    public_only = not request.user.is_authenticated and not request.__dict__.get("_api_token")
-    queryset = get_matching_tags(q)
-    public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
-    if public_only:
-        queryset = queryset.filter(snapshot_set__id__in=public_snapshots.values("id")).distinct()
-    tags = list(queryset[: 50 if not q else 20])
-    add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)
-
-    return {
-        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": tag.__dict__.get("num_snapshots", 0)} for tag in tags],
-    }
-
-
-@router.post("/tags/create/", response=TagCreateResponseSchema, url_name="tags_create")
-def tags_create(request: HttpRequest, data: TagCreateSchema):
-    """Create a new tag or return existing one."""
-    try:
-        tag, created = Tag.get_or_create_by_name(
-            data.name,
-            created_by=request.user if request.user.is_authenticated else None,
-        )
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-        "created": created,
-    }
-
-
-@router.post("/tag/{tag_id}/rename", response=TagUpdateResponseSchema, url_name="rename_tag")
-def rename_tag(request: HttpRequest, tag_id: int, data: TagUpdateSchema):
-    try:
-        tag = get_tag_by_ref(tag_id).rename(data.name)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-    }
-
-
-@router.delete("/tag/{tag_id}", response=TagDeleteResponseSchema, url_name="delete_tag")
-def delete_tag(request: HttpRequest, tag_id: int):
-    try:
-        tag = get_tag_by_ref(tag_id)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-
-    deleted_count, _ = tag.delete()
-    return {
-        "success": True,
-        "tag_id": int(tag_id),
-        "deleted_count": deleted_count,
-    }
-
-
-@router.get("/tag/{tag_id}/urls.txt", url_name="tag_urls_export")
-def tag_urls_export(request: HttpRequest, tag_id: int):
-    try:
-        tag = get_tag_by_ref(tag_id)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-
-    response = HttpResponse(export_tag_urls(tag), content_type="text/plain; charset=utf-8")
-    response["Content-Disposition"] = f'attachment; filename="tag-{tag.slug}-urls.txt"'
-    return response
-
-
-@router.get("/tag/{tag_id}/snapshots.jsonl", url_name="tag_snapshots_export")
-def tag_snapshots_export(request: HttpRequest, tag_id: int):
-    try:
-        tag = get_tag_by_ref(tag_id)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-
-    response = HttpResponse(export_tag_snapshots_jsonl(tag), content_type="application/x-ndjson; charset=utf-8")
-    response["Content-Disposition"] = f'attachment; filename="tag-{tag.slug}-snapshots.jsonl"'
-    return response
-
-
-@router.post("/tags/add-to-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_add_to_snapshot")
-def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
-    """Add a tag to a snapshot. Creates the tag if it doesn't exist."""
-    snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
-
-    # Get or create the tag
-    if data.tag_name:
-        try:
-            tag, _ = Tag.get_or_create_by_name(
-                data.tag_name,
-                created_by=request.user if request.user.is_authenticated else None,
-            )
-        except ValueError as err:
-            raise HttpError(400, str(err)) from err
-    elif data.tag_id:
-        try:
-            tag = get_tag_by_ref(data.tag_id)
-        except Tag.DoesNotExist:
-            raise HttpError(404, "Tag not found")
-    else:
-        raise HttpError(400, "Either tag_name or tag_id is required")
-
-    snapshot.add_tag_ids([tag.pk])
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-    }
-
-
-@router.post("/tags/remove-from-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_remove_from_snapshot")
-def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
-    """Remove a tag from a snapshot."""
-    snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
-
-    # Get the tag
-    if data.tag_id:
-        try:
-            tag = Tag.objects.get(pk=data.tag_id)
-        except Tag.DoesNotExist:
-            raise HttpError(404, "Tag not found")
-    elif data.tag_name:
-        try:
-            tag = Tag.objects.get(name__iexact=data.tag_name.strip())
-        except Tag.DoesNotExist:
-            raise HttpError(404, "Tag not found")
-    else:
-        raise HttpError(400, "Either tag_name or tag_id is required")
-
-    snapshot.remove_tag_ids([tag.pk])
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-    }

archivebox/api/v1_crawls.py

@@ -7,10 +7,9 @@ from django.http import FileResponse, HttpRequest
 from django.shortcuts import redirect
 from django.utils import timezone
 
+from django.contrib.auth import get_user_model
 from django.contrib.auth.models import User
 
-from archivebox.api.schemas import OwnedObjectSchema
-
 from ninja import Router, Schema
 from ninja.errors import HttpError
 
@@ -30,13 +29,15 @@ from .auth import API_AUTH_METHODS, authenticated_user_from_request
 router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
 
 
-class CrawlSchema(OwnedObjectSchema):
+class CrawlSchema(Schema):
     TYPE: str = "crawls.models.Crawl"
 
     id: UUID
 
     modified_at: datetime
     created_at: datetime
+    created_by_id: str
+    created_by_username: str
 
     status: str
     retry_at: datetime | None
@@ -47,6 +48,17 @@ class CrawlSchema(OwnedObjectSchema):
     tags_str: str
     config: dict
 
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by_id)
+
+    @staticmethod
+    def resolve_created_by_username(obj):
+        user_model = get_user_model()
+        user = user_model.objects.get(id=obj.created_by_id)
+        username = user.username
+        return username if isinstance(username, str) else str(user)
+
     @staticmethod
     def resolve_config(obj):
         # Redact credential values so REST responses can never leak the raw

archivebox/api/v1_personas.py

@@ -2,13 +2,11 @@ __package__ = "archivebox.api"
 
 import json
 from datetime import datetime
-from typing import Any
+from typing import Any, Literal
 from uuid import UUID
 
 from django.db.models import Q
 from django.http import HttpRequest
-from archivebox.api.schemas import OwnedObjectSchema
-
 from ninja import Router, Schema
 from ninja.pagination import paginate
 from pydantic import Field
@@ -25,6 +23,7 @@ class PersonaBrowserSettingsSchema(Schema):
     user_agent: str = ""
     viewport_size: str = ""
     viewport_device_scale_factor: float | None = None
+    color_scheme: Literal["", "light", "dark"] = ""
     language: str = ""
     timezone: str = ""
     geolocation: dict[str, Any] | None = None
@@ -38,13 +37,23 @@ class PersonaSyncSchema(Schema):
     auth_json: dict[str, Any] = Field(default_factory=dict)
 
 
-class PersonaSchema(OwnedObjectSchema):
+class PersonaSchema(Schema):
     TYPE: str = "personas.models.Persona"
     id: UUID
     name: str
     created_at: datetime
+    created_by_id: str
+    created_by_username: str
     config: dict[str, Any] | None
 
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by.pk)
+
+    @staticmethod
+    def resolve_created_by_username(obj) -> str:
+        return obj.created_by.username
+
     @staticmethod
     def resolve_config(obj):
         # Redact credential values so REST responses don't leak the raw
@@ -86,6 +95,8 @@ def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrows
         )
     if settings.viewport_device_scale_factor is not None:
         config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor
+    if "color_scheme" in settings.model_fields_set:
+        config["BROWSER_COLOR_SCHEME"] = settings.color_scheme
     if settings.language:
         config["BROWSER_LANGUAGE"] = settings.language
     if settings.timezone:
@@ -142,12 +153,12 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     persona.ensure_dirs()
 
     cookies_written = False
-    if payload.cookies_txt.strip():
+    if "cookies_txt" in payload.model_fields_set:
         (persona.path / "cookies.txt").write_text(payload.cookies_txt)
         cookies_written = True
 
     auth_written = False
-    if payload.auth_json:
+    if "auth_json" in payload.model_fields_set:
         (persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n")
         auth_written = True
 

archivebox/base_models/admin.py

@@ -13,16 +13,10 @@ from django.db import DatabaseError, models
 from django.forms.renderers import BaseRenderer
 from django.http import HttpRequest, QueryDict
 from django.urls import path, register_converter
-from django.utils.html import format_html
 from django.utils.safestring import SafeString, mark_safe
 from django_object_actions import DjangoObjectActions
 
 
-def card_fieldset(title: str | None, fields: tuple, *, wide: bool = False, **options):
-    """Build a Django fieldset using the shared card layout."""
-    return title, {"fields": fields, "classes": ("card", "wide") if wide else ("card",), **options}
-
-
 class HexUUIDConverter:
     """URL path converter that canonicalizes UUIDs to their 32-char hex form.
 
@@ -87,11 +81,6 @@ class KeyValueWidget(forms.Widget):
             options: dict[str, ConfigOption] = {}
             for key, metadata in config_field_metadata().items():
                 option_type = metadata.get("type", "string")
-                # Core fields expose Python type names; plugins use JSON Schema.
-                # The editor should apply the same typed validation to both.
-                type_names = {"bool": "boolean", "int": "integer", "float": "number", "str": "string", "dict": "object", "list": "array"}
-                if isinstance(option_type, str):
-                    option_type = type_names.get(option_type, option_type)
                 option: ConfigOption = {
                     "plugin": str(metadata.get("plugin", "archivebox")),
                     "type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
@@ -129,36 +118,650 @@ class KeyValueWidget(forms.Widget):
         attrs: Mapping[str, str] | None = None,
         renderer: BaseRenderer | None = None,
     ) -> SafeString:
-        from django.template.loader import render_to_string
+        data = self._parse_value(value)
 
         widget_id = attrs.get("id", name) if attrs else name
-        options = self._get_config_options()
-        rows = [
-            mark_safe(self._render_row(widget_id, key, val if isinstance(val, str) else json.dumps(val)))
-            for key, val in self._parse_value(value).items()
-        ]
-        empty_row = mark_safe(self._render_row(widget_id, "", ""))
-        # JSON is embedded in a script, so escape HTML delimiters without changing data.
-        metadata = json.dumps(options).translate({ord("<"): r"\u003C", ord(">"): r"\u003E", ord("&"): r"\u0026"})
-        return mark_safe(
-            render_to_string(
-                "admin/widgets/config.html",
-                {
-                    "name": name,
-                    "widget_id": widget_id,
-                    "datalist_options": mark_safe(
-                        "\n".join(
-                            f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
-                            for key, opt in sorted(options.items())
-                        ),
-                    ),
-                    "config_meta_json": metadata,
-                    "rows": [*rows, empty_row],
-                    "empty_row": empty_row,
-                },
-            ),
+        config_options = self._get_config_options()
+
+        # Build datalist options
+        datalist_options = "\n".join(
+            f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
+            for key, opt in sorted(config_options.items())
         )
 
+        # Build config metadata as JSON for JS
+        config_meta_json = json.dumps(config_options)
+
+        html = f'''
+        <div id="{widget_id}_container" class="key-value-editor" style="width: 100%; max-width: none;">
+            <datalist id="{widget_id}_keys">
+                {datalist_options}
+            </datalist>
+            <div id="{widget_id}_rows" class="key-value-rows">
+        '''
+
+        # Render existing key-value pairs
+        for key, val in data.items():
+            val_str = json.dumps(val) if not isinstance(val, str) else val
+            html += self._render_row(widget_id, key, val_str)
+
+        # Always add one empty row for new entries
+        html += self._render_row(widget_id, "", "")
+
+        html += f'''
+            </div>
+            <div style="display: flex; gap: 8px; align-items: center; margin-top: 8px;">
+                <button type="button" onclick="addKeyValueRow_{widget_id}()"
+                        style="padding: 4px 12px; cursor: pointer; background: #417690; color: white; border: none; border-radius: 4px;">
+                    + Add Row
+                </button>
+            </div>
+            <input type="hidden" name="{name}" id="{widget_id}" value="">
+            <script>
+                (function() {{
+                    var configMeta_{widget_id} = {config_meta_json};
+                    var rowCounter_{widget_id} = 0;
+
+                    function stringifyValue_{widget_id}(value) {{
+                        return typeof value === 'string' ? value : JSON.stringify(value);
+                    }}
+
+                    function getTypes_{widget_id}(meta) {{
+                        if (!meta || meta.type === undefined || meta.type === null) {{
+                            return [];
+                        }}
+                        return Array.isArray(meta.type) ? meta.type : [meta.type];
+                    }}
+
+                    function getMetaForKey_{widget_id}(key) {{
+                        if (!key) {{
+                            return null;
+                        }}
+
+                        var explicitMeta = configMeta_{widget_id}[key];
+                        if (explicitMeta) {{
+                            return Object.assign({{ key: key }}, explicitMeta);
+                        }}
+
+                        if (key.endsWith('_BINARY')) {{
+                            return {{
+                                key: key,
+                                plugin: 'custom',
+                                type: 'string',
+                                default: '',
+                                description: 'Path to binary executable',
+                            }};
+                        }}
+
+                        if (isRegexConfigKey_{widget_id}(key)) {{
+                            return {{
+                                key: key,
+                                plugin: 'custom',
+                                type: 'string',
+                                default: '',
+                                description: 'Regex pattern list',
+                            }};
+                        }}
+
+                        return null;
+                    }}
+
+                    function describeMeta_{widget_id}(meta) {{
+                        if (!meta) {{
+                            return '';
+                        }}
+
+                        var details = '';
+                        if (Array.isArray(meta.enum) && meta.enum.length) {{
+                            details = 'Allowed: ' + meta.enum.map(stringifyValue_{widget_id}).join(', ');
+                        }} else {{
+                            var types = getTypes_{widget_id}(meta);
+                            if (types.length) {{
+                                details = 'Expected: ' + types.join(' or ');
+                            }}
+                        }}
+
+                        if (meta.minimum !== undefined || meta.maximum !== undefined) {{
+                            var bounds = [];
+                            if (meta.minimum !== undefined) bounds.push('min ' + meta.minimum);
+                            if (meta.maximum !== undefined) bounds.push('max ' + meta.maximum);
+                            details += (details ? ' ' : '') + '(' + bounds.join(', ') + ')';
+                        }}
+
+                        return [meta.description || '', details].filter(Boolean).join(' ');
+                    }}
+
+                    function getExampleInput_{widget_id}(key, meta) {{
+                        var types = getTypes_{widget_id}(meta);
+                        if (key.endsWith('_BINARY')) {{
+                            return 'Example: wget or /usr/bin/wget';
+                        }}
+                        if (key.endsWith('_ARGS_EXTRA') || key.endsWith('_ARGS')) {{
+                            return 'Example: ["--extra-arg"]';
+                        }}
+                        if (types.includes('array')) {{
+                            return 'Example: ["value"]';
+                        }}
+                        if (types.includes('object')) {{
+                            return 'Example: {{"key": "value"}}';
+                        }}
+                        return '';
+                    }}
+
+                    function isRegexConfigKey_{widget_id}(key) {{
+                        return key === 'URL_ALLOWLIST' ||
+                            key === 'URL_DENYLIST' ||
+                            key.endsWith('_PATTERN') ||
+                            key.includes('REGEX');
+                    }}
+
+                    function isSimpleFilterPattern_{widget_id}(pattern) {{
+                        return /^[\\w.*:-]+$/.test(pattern);
+                    }}
+
+                    function validateRegexPattern_{widget_id}(pattern) {{
+                        if (!pattern || isSimpleFilterPattern_{widget_id}(pattern)) {{
+                            return '';
+                        }}
+
+                        try {{
+                            new RegExp(pattern);
+                        }} catch (error) {{
+                            return error && error.message ? error.message : 'Invalid regex';
+                        }}
+                        return '';
+                    }}
+
+                    function validateRegexConfig_{widget_id}(key, raw, typeName) {{
+                        if (typeName === 'object') {{
+                            var parsed;
+                            try {{
+                                parsed = JSON.parse(raw);
+                            }} catch (error) {{
+                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
+                            }}
+                            if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {{
+                                return {{ ok: false, value: parsed, message: 'Must be a JSON object' }};
+                            }}
+                            for (var regexKey in parsed) {{
+                                var objectRegexError = validateRegexPattern_{widget_id}(regexKey);
+                                if (objectRegexError) {{
+                                    return {{ ok: false, value: parsed, message: 'Invalid regex key "' + regexKey + '": ' + objectRegexError }};
+                                }}
+                            }}
+                            return {{ ok: true, value: parsed, message: '' }};
+                        }}
+
+                        var patterns = raw.split(/[\\n,]+/).map(function(pattern) {{
+                            return pattern.trim();
+                        }}).filter(Boolean);
+                        for (var i = 0; i < patterns.length; i++) {{
+                            var regexError = validateRegexPattern_{widget_id}(patterns[i]);
+                            if (regexError) {{
+                                return {{ ok: false, value: raw, message: 'Invalid regex "' + patterns[i] + '": ' + regexError }};
+                            }}
+                        }}
+                        return {{ ok: true, value: raw, message: '' }};
+                    }}
+
+                    function validateBinaryValue_{widget_id}(raw) {{
+                        if (!raw) {{
+                            return {{ ok: true, value: raw, message: '' }};
+                        }}
+
+                        if (/['"`]/.test(raw)) {{
+                            return {{ ok: false, value: raw, message: 'Binary paths cannot contain quotes' }};
+                        }}
+
+                        if (/[;&|<>$(){{}}\\[\\]!]/.test(raw)) {{
+                            return {{ ok: false, value: raw, message: 'Binary paths can only be a binary name or absolute path' }};
+                        }}
+
+                        if (raw.startsWith('/')) {{
+                            if (/^[A-Za-z0-9_./+\\- ]+$/.test(raw)) {{
+                                return {{ ok: true, value: raw, message: '' }};
+                            }}
+                            return {{ ok: false, value: raw, message: 'Absolute paths may only contain path-safe characters' }};
+                        }}
+
+                        if (/^[A-Za-z0-9_.+-]+$/.test(raw)) {{
+                            return {{ ok: true, value: raw, message: '' }};
+                        }}
+
+                        return {{ ok: false, value: raw, message: 'Enter a binary name like wget or an absolute path like /usr/bin/wget' }};
+                    }}
+
+                    function parseValue_{widget_id}(raw) {{
+                        try {{
+                            if (raw === 'true') return true;
+                            if (raw === 'false') return false;
+                            if (raw === 'null') return null;
+                            if (raw !== '' && !isNaN(raw)) return Number(raw);
+                            if ((raw.startsWith('{{') && raw.endsWith('}}')) ||
+                                (raw.startsWith('[') && raw.endsWith(']')) ||
+                                (raw.startsWith('"') && raw.endsWith('"'))) {{
+                                return JSON.parse(raw);
+                            }}
+                        }} catch (error) {{
+                            return raw;
+                        }}
+                        return raw;
+                    }}
+
+                    function sameValue_{widget_id}(left, right) {{
+                        return left === right || JSON.stringify(left) === JSON.stringify(right);
+                    }}
+
+                    function parseTypedValue_{widget_id}(raw, typeName, meta) {{
+                        var numberValue;
+                        var parsed;
+
+                        if (typeName && meta && meta.key && isRegexConfigKey_{widget_id}(meta.key)) {{
+                            return validateRegexConfig_{widget_id}(meta.key, raw, typeName);
+                        }}
+
+                        if (typeName === 'string' && meta && meta.key && meta.key.endsWith('_BINARY')) {{
+                            return validateBinaryValue_{widget_id}(raw);
+                        }}
+
+                        if (typeName === 'string') {{
+                            if (meta.pattern) {{
+                                try {{
+                                    if (!(new RegExp(meta.pattern)).test(raw)) {{
+                                        return {{ ok: false, value: raw, message: 'Must match pattern ' + meta.pattern }};
+                                    }}
+                                }} catch (error) {{}}
+                            }}
+                            return {{ ok: true, value: raw, message: '' }};
+                        }}
+
+                        if (typeName === 'integer') {{
+                            if (!/^-?\\d+$/.test(raw)) {{
+                                return {{ ok: false, value: raw, message: 'Must be an integer' }};
+                            }}
+                            numberValue = Number(raw);
+                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
+                            }}
+                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
+                            }}
+                            return {{ ok: true, value: numberValue, message: '' }};
+                        }}
+
+                        if (typeName === 'number') {{
+                            if (raw === '' || isNaN(raw)) {{
+                                return {{ ok: false, value: raw, message: 'Must be a number' }};
+                            }}
+                            numberValue = Number(raw);
+                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
+                            }}
+                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
+                            }}
+                            return {{ ok: true, value: numberValue, message: '' }};
+                        }}
+
+                        if (typeName === 'boolean') {{
+                            var lowered = raw.toLowerCase();
+                            if (lowered === 'true' || raw === '1') return {{ ok: true, value: true, message: '' }};
+                            if (lowered === 'false' || raw === '0') return {{ ok: true, value: false, message: '' }};
+                            return {{ ok: false, value: raw, message: 'Must be true or false' }};
+                        }}
+
+                        if (typeName === 'null') {{
+                            return raw === 'null'
+                                ? {{ ok: true, value: null, message: '' }}
+                                : {{ ok: false, value: raw, message: 'Must be null' }};
+                        }}
+
+                        if (typeName === 'array' || typeName === 'object') {{
+                            try {{
+                                parsed = JSON.parse(raw);
+                            }} catch (error) {{
+                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
+                            }}
+
+                            if (typeName === 'array' && Array.isArray(parsed)) {{
+                                return {{ ok: true, value: parsed, message: '' }};
+                            }}
+                            if (typeName === 'object' && parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {{
+                                return {{ ok: true, value: parsed, message: '' }};
+                            }}
+
+                            return {{
+                                ok: false,
+                                value: parsed,
+                                message: typeName === 'array' ? 'Must be a JSON array' : 'Must be a JSON object',
+                            }};
+                        }}
+
+                        return {{ ok: true, value: parseValue_{widget_id}(raw), message: '' }};
+                    }}
+
+                    function validateValueAgainstMeta_{widget_id}(raw, meta) {{
+                        if (!meta || raw === '') {{
+                            return {{ state: 'neutral', value: raw, message: '' }};
+                        }}
+
+                        var enumValues = Array.isArray(meta.enum) ? meta.enum : [];
+                        var types = getTypes_{widget_id}(meta);
+                        if (!types.length) {{
+                            types = ['string'];
+                        }}
+
+                        var error = 'Invalid value';
+                        for (var i = 0; i < types.length; i++) {{
+                            var candidate = parseTypedValue_{widget_id}(raw, types[i], meta);
+                            if (!candidate.ok) {{
+                                error = candidate.message || error;
+                                continue;
+                            }}
+                            if (enumValues.length && !enumValues.some(function(enumValue) {{
+                                return sameValue_{widget_id}(enumValue, candidate.value) || stringifyValue_{widget_id}(enumValue) === raw;
+                            }})) {{
+                                error = 'Must be one of: ' + enumValues.map(stringifyValue_{widget_id}).join(', ');
+                                continue;
+                            }}
+                            return {{ state: 'valid', value: candidate.value, message: '' }};
+                        }}
+
+                        return {{ state: 'invalid', value: raw, message: error }};
+                    }}
+
+                    function ensureRowId_{widget_id}(row) {{
+                        if (!row.dataset.rowId) {{
+                            row.dataset.rowId = String(rowCounter_{widget_id}++);
+                        }}
+                        return row.dataset.rowId;
+                    }}
+
+                    function setRowHelp_{widget_id}(row) {{
+                        var keyInput = row.querySelector('.kv-key');
+                        var help = row.querySelector('.kv-help');
+                        if (!keyInput || !help) {{
+                            return;
+                        }}
+
+                        var key = keyInput.value.trim();
+                        if (!key) {{
+                            help.textContent = '';
+                            return;
+                        }}
+
+                        var meta = getMetaForKey_{widget_id}(key);
+                        if (meta) {{
+                            var extra = isRegexConfigKey_{widget_id}(key)
+                                ? ((meta.type === 'object' || (Array.isArray(meta.type) && meta.type.includes('object')))
+                                    ? ' Expected: JSON object with regex keys.'
+                                    : ' Expected: valid regex.')
+                                : '';
+                            var example = getExampleInput_{widget_id}(key, meta);
+                            help.textContent = [describeMeta_{widget_id}(meta) + extra, example].filter(Boolean).join(' ');
+                        }} else {{
+                            help.textContent = 'Custom key';
+                        }}
+                    }}
+
+                    function configureValueInput_{widget_id}(row) {{
+                        var keyInput = row.querySelector('.kv-key');
+                        var valueInput = row.querySelector('.kv-value');
+                        var datalist = row.querySelector('.kv-value-options');
+                        if (!keyInput || !valueInput || !datalist) {{
+                            return;
+                        }}
+
+                        var rowId = ensureRowId_{widget_id}(row);
+                        datalist.id = '{widget_id}_value_options_' + rowId;
+
+                        var meta = getMetaForKey_{widget_id}(keyInput.value.trim());
+                        var enumValues = Array.isArray(meta && meta.enum) ? meta.enum : [];
+                        var types = getTypes_{widget_id}(meta);
+                        if (!enumValues.length && types.includes('boolean')) {{
+                            enumValues = ['True', 'False'];
+                        }}
+                        if (enumValues.length) {{
+                            datalist.innerHTML = enumValues.map(function(enumValue) {{
+                                return '<option value="' + stringifyValue_{widget_id}(enumValue).replace(/"/g, '&quot;') + '"></option>';
+                            }}).join('');
+                            valueInput.setAttribute('list', datalist.id);
+                        }} else {{
+                            datalist.innerHTML = '';
+                            valueInput.removeAttribute('list');
+                        }}
+                    }}
+
+                    function setValueValidationState_{widget_id}(input, state, message) {{
+                        if (!input) {{
+                            return;
+                        }}
+
+                        if (state === 'valid') {{
+                            input.style.borderColor = '#2da44e';
+                            input.style.boxShadow = '0 0 0 1px rgba(45, 164, 78, 0.18)';
+                            input.style.backgroundColor = '#f6ffed';
+                        }} else if (state === 'invalid') {{
+                            input.style.borderColor = '#cf222e';
+                            input.style.boxShadow = '0 0 0 1px rgba(207, 34, 46, 0.18)';
+                            input.style.backgroundColor = '#fff8f8';
+                        }} else {{
+                            input.style.borderColor = '#ccc';
+                            input.style.boxShadow = 'none';
+                            input.style.backgroundColor = '';
+                        }}
+                        input.title = message || '';
+                    }}
+
+                    function applyValueValidation_{widget_id}(row) {{
+                        var keyInput = row.querySelector('.kv-key');
+                        var valueInput = row.querySelector('.kv-value');
+                        if (!keyInput || !valueInput) {{
+                            return;
+                        }}
+
+                        var key = keyInput.value.trim();
+                        if (!key) {{
+                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
+                            return;
+                        }}
+
+                        var meta = getMetaForKey_{widget_id}(key);
+                        if (!meta) {{
+                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
+                            return;
+                        }}
+
+                        var validation = validateValueAgainstMeta_{widget_id}(valueInput.value.trim(), meta);
+                        setValueValidationState_{widget_id}(valueInput, validation.state, validation.message);
+                    }}
+
+                    function coerceValueForStorage_{widget_id}(key, raw) {{
+                        var meta = getMetaForKey_{widget_id}(key);
+                        if (!meta) {{
+                            return parseValue_{widget_id}(raw);
+                        }}
+
+                        var validation = validateValueAgainstMeta_{widget_id}(raw, meta);
+                        return validation.state === 'valid' ? validation.value : raw;
+                    }}
+
+                    function initializeRows_{widget_id}() {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
+                            ensureRowId_{widget_id}(row);
+                            configureValueInput_{widget_id}(row);
+                            setRowHelp_{widget_id}(row);
+                            applyValueValidation_{widget_id}(row);
+                        }});
+                    }}
+
+                    function updateHiddenField_{widget_id}() {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        var rows = container.querySelectorAll('.key-value-row');
+                        var result = {{}};
+                        rows.forEach(function(row) {{
+                            var keyInput = row.querySelector('.kv-key');
+                            var valInput = row.querySelector('.kv-value');
+                            if (keyInput && valInput && keyInput.value.trim()) {{
+                                var key = keyInput.value.trim();
+                                var val = valInput.value.trim();
+                                result[key] = coerceValueForStorage_{widget_id}(key, val);
+                            }}
+                        }});
+                        document.getElementById('{widget_id}').value = JSON.stringify(result);
+                    }}
+
+                    window.addKeyValueRow_{widget_id} = function() {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        var newRow = document.createElement('div');
+                        newRow.className = 'key-value-row';
+                        newRow.style.cssText = 'margin-bottom: 6px;';
+                        newRow.innerHTML = '<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">' +
+                            '<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
+                            'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
+                            '<input type="text" class="kv-value" placeholder="value" ' +
+                            'style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
+                            '<datalist class="kv-value-options"></datalist>' +
+                            '<button type="button" onclick="removeKeyValueRow_{widget_id}(this)" ' +
+                            'style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>' +
+                            '</div>' +
+                            '<div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>';
+                        container.appendChild(newRow);
+                        ensureRowId_{widget_id}(newRow);
+                        configureValueInput_{widget_id}(newRow);
+                        setRowHelp_{widget_id}(newRow);
+                        applyValueValidation_{widget_id}(newRow);
+                        updateHiddenField_{widget_id}();
+                        newRow.querySelector('.kv-key').focus();
+                    }};
+
+                    window.removeKeyValueRow_{widget_id} = function(btn) {{
+                        var row = btn.closest('.key-value-row');
+                        row.remove();
+                        updateHiddenField_{widget_id}();
+                    }};
+
+                    window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
+
+                    function configRowForKey_{widget_id}(key) {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        if (!container) {{
+                            return null;
+                        }}
+                        var match = null;
+                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
+                            if (match) {{ return; }}
+                            var keyInput = row.querySelector('.kv-key');
+                            if (keyInput && keyInput.value.trim() === key) {{
+                                match = row;
+                            }}
+                        }});
+                        if (!match) {{
+                            window.addKeyValueRow_{widget_id}();
+                            var rows = container.querySelectorAll('.key-value-row');
+                            match = rows[rows.length - 1];
+                            var keyInput = match.querySelector('.kv-key');
+                            if (keyInput) {{
+                                keyInput.value = key;
+                                keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
+                            }}
+                        }}
+                        return match;
+                    }}
+
+                    function prefillConfigFromQuery_{widget_id}() {{
+                        var params = new URLSearchParams(window.location.search);
+                        var consumedConfigKeys = [];
+                        params.forEach(function(value, key) {{
+                            if (!/^[A-Z][A-Z0-9_]*$/.test(key) || !configMeta_{widget_id}[key]) {{
+                                return;
+                            }}
+                            consumedConfigKeys.push(key);
+                            var match = configRowForKey_{widget_id}(key);
+                            var valueInput = match && match.querySelector('.kv-value');
+                            if (valueInput) {{
+                                valueInput.value = value;
+                                valueInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
+                            }}
+                        }});
+                        updateHiddenField_{widget_id}();
+                        if (consumedConfigKeys.length) {{
+                            var cleanUrl = new URL(window.location.href);
+                            consumedConfigKeys.forEach(function(key) {{ cleanUrl.searchParams.delete(key); }});
+                            window.history.replaceState(null, '', cleanUrl.pathname + cleanUrl.search + cleanUrl.hash);
+                        }}
+                    }}
+
+                    function focusConfigKeyFromHash_{widget_id}() {{
+                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
+                        // to (or creates) the matching row in this editor. Used by
+                        // the setup wizard and live-config detail pages.
+                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
+                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
+                            return;
+                        }}
+                        var match = configRowForKey_{widget_id}(hash);
+                        if (!match) {{
+                            return;
+                        }}
+                        match.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
+                        var prevOutline = match.style.outline;
+                        match.style.outline = '2px solid #f59e0b';
+                        match.style.outlineOffset = '2px';
+                        match.style.transition = 'outline 1.2s ease-out';
+                        setTimeout(function() {{
+                            match.style.outline = prevOutline || 'none';
+                        }}, 1400);
+                        var valueInput = match.querySelector('.kv-value');
+                        if (valueInput) {{
+                            valueInput.focus();
+                            try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}}
+                        }}
+                    }}
+
+                    // Initialize on load
+                    document.addEventListener('DOMContentLoaded', function() {{
+                        initializeRows_{widget_id}();
+                        prefillConfigFromQuery_{widget_id}();
+                        updateHiddenField_{widget_id}();
+                        focusConfigKeyFromHash_{widget_id}();
+                    }});
+                    // Also run immediately in case DOM is already ready
+                    if (document.readyState !== 'loading') {{
+                        initializeRows_{widget_id}();
+                        prefillConfigFromQuery_{widget_id}();
+                        updateHiddenField_{widget_id}();
+                        focusConfigKeyFromHash_{widget_id}();
+                    }}
+
+                    window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id});
+
+                    // Update on any input change
+                    var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
+
+                    rowsEl_{widget_id}.addEventListener('input', function(event) {{
+                        var row = event.target.closest('.key-value-row');
+                        if (!row) {{
+                            return;
+                        }}
+
+                        if (event.target.classList.contains('kv-key')) {{
+                            configureValueInput_{widget_id}(row);
+                            setRowHelp_{widget_id}(row);
+                        }}
+
+                        if (event.target.classList.contains('kv-key') || event.target.classList.contains('kv-value')) {{
+                            applyValueValidation_{widget_id}(row);
+                            updateHiddenField_{widget_id}();
+                        }}
+                    }});
+                }})();
+            </script>
+        </div>
+        '''
+        return mark_safe(html)
+
     def _render_row(self, widget_id: str, key: str, value: str) -> str:
         from archivebox.config.common import is_sensitive_config_key
 
@@ -269,28 +872,6 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
     show_search_mode_selector = False
     change_form_template = "admin/archivebox_change_form.html"
 
-    @admin.display(description="Health", ordering="health")
-    def health_display(self, obj):
-        h = obj.health
-        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
-        return format_html('<span style="color: {};">{}</span>', color, h)
-
-    def get_ordering_fields(self, request):
-        ordering = request.GET.get("o")
-        if not ordering:
-            return set()
-        fields = set()
-        for part in ordering.split("."):
-            if not part:
-                continue
-            try:
-                idx = abs(int(part)) - 1
-            except ValueError:
-                continue
-            if 0 <= idx < len(self.list_display):
-                fields.add(self.list_display[idx])
-        return fields
-
     def get_admin_toolbar_actions(self, request, obj):
         """Return extra action button dicts for the shared change-form toolbar.
 

archivebox/base_models/models.py

@@ -4,7 +4,7 @@ __package__ = "archivebox.base_models"
 
 import json
 import shutil
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
@@ -23,10 +23,6 @@ from django_stubs_ext.db.models import TypedModelMeta
 from archivebox.config import CONSTANTS
 
 
-if TYPE_CHECKING:
-    from archivebox.config.common import ArchiveBoxBaseConfig
-
-
 def normalize_config_json_values(config: Any) -> Any:
     if not isinstance(config, dict):
         return config
@@ -59,6 +55,17 @@ def get_or_create_system_user_pk(username="system"):
     return user.pk
 
 
+class AutoDateTimeField(models.DateTimeField):
+    """DateTimeField that automatically updates on save (legacy compatibility)."""
+
+    def pre_save(self, model_instance, add):
+        if add or self.attname not in model_instance.__dict__ or not model_instance.__dict__[self.attname]:
+            value = timezone.now()
+            setattr(model_instance, self.attname, value)
+            return value
+        return super().pre_save(model_instance, add)
+
+
 class ModelWithUUID(models.Model):
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -131,7 +138,6 @@ class ModelWithHealthStats(models.Model):
 class ModelWithConfig(models.Model):
     """Mixin for models with a JSON config field."""
 
-    _runtime_config: "ArchiveBoxBaseConfig"
     config = models.JSONField(default=dict, null=True, blank=True, editable=True)
 
     class Meta(TypedModelMeta):

archivebox/cli/__init__.py

@@ -155,13 +155,10 @@ def cli(ctx, help=False):
     wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
     if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
         try:
-            if subcommand == "server":
-                run_in_debug = "--reload" in sys.argv or os.environ.get("DEBUG") in ("1", "true", "True", "TRUE", "yes")
-                if run_in_debug:
-                    os.environ["ARCHIVEBOX_RUNSERVER"] = "1"
-                    if "--reload" in sys.argv:
-                        os.environ["ARCHIVEBOX_AUTORELOAD"] = "1"
-
+            # RUNSERVER_WORKER sets the web-worker markers on its own child.
+            # Setting them on the server parent leaks them through supervisord
+            # into runners waiting for takeover, which then register themselves
+            # as web workers and trigger spurious runner_watch reloads.
             from archivebox.config.django import setup_django
             from archivebox.misc.checks import check_data_folder, check_migrations
 

archivebox/cli/archivebox_add.py

@@ -6,6 +6,7 @@ __command__ = "archivebox add"
 
 import sys
 import os
+import signal
 from pathlib import Path
 
 from typing import Any, TYPE_CHECKING
@@ -227,6 +228,10 @@ def add(
 
         assert command is not None
         exit_code = 0
+        interactive_interrupts = sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())
+
+        def interrupt_hook() -> None:
+            raise KeyboardInterrupt
 
         def crawl_is_complete() -> bool:
             crawl.refresh_from_db(fields=["status"])
@@ -234,7 +239,13 @@ def add(
 
         try:
             try:
-                with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
+                with (
+                    foreground_shutdown_signals(
+                        first_signal_message=None,
+                        interrupt_handlers={signal.SIGINT: interrupt_hook},
+                    ),
+                    foreground_parent_watchdog(),
+                ):
                     while True:
                         standby = standby_until_foreground_runner_needed(
                             command,
@@ -246,7 +257,7 @@ def add(
                         exit_code = run_runner_worker(
                             ["--crawl-id", str(crawl.id)],
                             name=f"worker_runner_add_{os.getpid()}",
-                            interactive_interrupts=True,
+                            interactive_interrupts=interactive_interrupts,
                             config=get_config(crawl=crawl),
                         )
                         crawl.refresh_from_db(fields=["status", "retry_at"])
@@ -311,9 +322,13 @@ def add(
 
             admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
 
-            print("\n[bold]crawl output saved to:[/bold]")
-            print(f"  {rel_output_str}")
-            print(f"  {admin_url}")
+            if snapshots_count:
+                print("\n[bold]snapshot output saved to:[/bold]")
+                for snapshot in crawl.snapshot_set.order_by("created_at").iterator():
+                    click.echo(f"  {snapshot.output_dir}")
+            print("\n[bold]crawl logs and setup:[/bold]")
+            click.echo(f"  {rel_output_str}")
+            click.echo(f"  {admin_url}")
             print(f"\n[bold]total urls snapshotted:[/bold] {snapshots_count}")
             print(f"[bold]total size:[/bold] {total_size}")
             print(f"[bold]total time:[/bold] {duration_str}")

archivebox/cli/archivebox_archiveresult.py

@@ -72,7 +72,7 @@ def create_archiveresults(
     """
     from archivebox.config.common import get_config
     from archivebox.plugins.hooks import discover_hooks
-    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT
+    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
     from archivebox.core.models import Snapshot
 
     is_tty = sys.stdout.isatty()
@@ -99,9 +99,26 @@ def create_archiveresults(
         for record in records:
             record_type = record.get("type", "")
 
-            if record_type:
+            if record_type == TYPE_SNAPSHOT:
+                # Pass through the Snapshot record itself
                 pass_through_records.append(record)
-            if record.get("id") and record_type in ("", TYPE_SNAPSHOT):
+                if record.get("id"):
+                    snapshot_ids.append(record["id"])
+
+            elif record_type == TYPE_ARCHIVERESULT:
+                # ArchiveResult records: pass through if they have an id
+                if record.get("id"):
+                    pass_through_records.append(record)
+                # If no id, we could create it, but for now just pass through
+                else:
+                    pass_through_records.append(record)
+
+            elif record_type:
+                # Other typed records (Crawl, Tag, etc): pass through
+                pass_through_records.append(record)
+
+            elif record.get("id"):
+                # Untyped record with id - assume it's a snapshot ID
                 snapshot_ids.append(record["id"])
 
         # Output pass-through records first
@@ -152,20 +169,48 @@ def list_archiveresults(
     snapshot_id: str | None = None,
     limit: int | None = None,
 ) -> int:
-    """List archive results as JSONL, or formatted rows in a terminal."""
-    from archivebox.core.models import ArchiveResult
-    from archivebox.cli.cli_util import list_records, format_status
+    """
+    List ArchiveResults as JSONL with optional filters.
 
-    queryset = apply_filters(
-        ArchiveResult.objects.order_by("-start_ts"),
-        {"status": status, "plugin": plugin, "snapshot_id": snapshot_id},
-        limit=limit,
-    )
-    return list_records(
-        queryset,
-        plural="archive results",
-        render=lambda result: f"{format_status(result.status, 10)} {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
-    )
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.core.models import ArchiveResult
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = ArchiveResult.objects.all().order_by("-start_ts")
+
+    # Apply filters
+    filter_kwargs = {
+        "status": status,
+        "plugin": plugin,
+        "snapshot_id": snapshot_id,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for result in queryset:
+        if is_tty:
+            status_color = {
+                "queued": "yellow",
+                "started": "blue",
+                "succeeded": "green",
+                "failed": "red",
+                "skipped": "dim",
+                "noresults": "dim",
+                "backoff": "magenta",
+            }.get(result.status, "dim")
+            rprint(
+                f"[{status_color}]{result.status:10}[/{status_color}] {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
+            )
+        else:
+            write_record(result.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} archive results[/dim]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -173,17 +218,54 @@ def list_archiveresults(
 # =============================================================================
 
 
-def update_archiveresults(status: str | None = None) -> int:
-    """Apply supplied fields to each JSONL-selected ArchiveResult."""
+def update_archiveresults(
+    status: str | None = None,
+) -> int:
+    """
+    Update ArchiveResults from stdin JSONL.
+
+    Reads ArchiveResult records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from archivebox.misc.jsonl import read_stdin, write_record
     from archivebox.core.models import ArchiveResult
-    from archivebox.cli.cli_util import update_records
 
-    def update(archiveresult):
-        if status:
-            archiveresult.status = status
-        archiveresult.save()
+    is_tty = sys.stdout.isatty()
 
-    return update_records(ArchiveResult, update, plural="archive results")
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        result_id = record.get("id")
+        if not result_id:
+            continue
+
+        try:
+            result = ArchiveResult.objects.get(id=result_id)
+
+            # Apply updates from CLI flags
+            if status:
+                result.status = status
+
+            result.save()
+            updated_count += 1
+
+            if not is_tty:
+                write_record(result.to_json())
+
+        except ArchiveResult.DoesNotExist:
+            rprint(f"[yellow]ArchiveResult not found: {result_id}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} archive results[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -192,19 +274,52 @@ def update_archiveresults(status: str | None = None) -> int:
 
 
 def delete_archiveresults(yes: bool = False, dry_run: bool = False) -> int:
-    """Delete archive results selected by stdin JSONL; --yes confirms, --dry-run previews."""
-    from archivebox.cli.cli_util import delete_records
+    """
+    Delete ArchiveResults from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
     from archivebox.core.models import ArchiveResult
 
-    return delete_records(
-        ArchiveResult,
-        label="archive result",
-        plural="archive results",
-        preview=lambda obj: f"[dim]{obj.id}[/dim] {obj.plugin} {obj.snapshot.url[:40]}",
-        yes=yes,
-        dry_run=dry_run,
-        preview_limit=10,
-    )
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    result_ids = [r.get("id") for r in records if r.get("id")]
+
+    if not result_ids:
+        rprint("[yellow]No valid archive result IDs in input[/yellow]", file=sys.stderr)
+        return 1
+
+    results = ArchiveResult.objects.filter(id__in=result_ids)
+    count = results.count()
+
+    if count == 0:
+        rprint("[yellow]No matching archive results found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} archive results (dry run)[/yellow]", file=sys.stderr)
+        for result in results[:10]:
+            rprint(f"  [dim]{result.id}[/dim] {result.plugin} {result.snapshot.url[:40]}", file=sys.stderr)
+        if count > 10:
+            rprint(f"  ... and {count - 10} more", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = results.delete()
+    rprint(f"[green]Deleted {deleted_count} archive results[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -222,9 +337,9 @@ def main():
 @click.option("--snapshot-id", help="Snapshot ID to create results for")
 @click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
 @click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-def create_cmd(**kwargs):
+def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
     """Emit Snapshot plugin extraction requests as JSONL."""
-    sys.exit(create_archiveresults(**kwargs))
+    sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
 
 
 @main.command("list")
@@ -232,24 +347,36 @@ def create_cmd(**kwargs):
 @click.option("--plugin", "-p", help="Filter by plugin name")
 @click.option("--snapshot-id", help="Filter by snapshot ID")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(**kwargs):
+def list_cmd(
+    status: str | None,
+    plugin: str | None,
+    snapshot_id: str | None,
+    limit: int | None,
+):
     """List ArchiveResults as JSONL."""
-    sys.exit(list_archiveresults(**kwargs))
+    sys.exit(
+        list_archiveresults(
+            status=status,
+            plugin=plugin,
+            snapshot_id=snapshot_id,
+            limit=limit,
+        ),
+    )
 
 
 @main.command("update")
 @click.option("--status", "-s", help="Set status")
-def update_cmd(**kwargs):
+def update_cmd(status: str | None):
     """Update ArchiveResults from stdin JSONL."""
-    sys.exit(update_archiveresults(**kwargs))
+    sys.exit(update_archiveresults(status=status))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(**kwargs):
+def delete_cmd(yes: bool, dry_run: bool):
     """Delete ArchiveResults from stdin JSONL."""
-    sys.exit(delete_archiveresults(**kwargs))
+    sys.exit(delete_archiveresults(yes=yes, dry_run=dry_run))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_binary.py

@@ -114,20 +114,37 @@ def list_binaries(
     version__icontains: str | None = None,
     limit: int | None = None,
 ) -> int:
-    """List binaries as JSONL, or formatted rows in a terminal."""
-    from archivebox.machine.models import Binary
-    from archivebox.cli.cli_util import list_records
+    """
+    List Binaries as JSONL with optional filters.
 
-    queryset = apply_filters(
-        Binary.objects.order_by("name", "-modified_at", "-created_at"),
-        {"name": name, "abspath__icontains": abspath__icontains, "version__icontains": version__icontains},
-        limit=limit,
-    )
-    return list_records(
-        queryset,
-        plural="binaries",
-        render=lambda binary: f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}",
-    )
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.machine.models import Binary
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Binary.objects.all().order_by("name", "-modified_at", "-created_at")
+
+    # Apply filters
+    filter_kwargs = {
+        "name": name,
+        "abspath__icontains": abspath__icontains,
+        "version__icontains": version__icontains,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for binary in queryset:
+        if is_tty:
+            rprint(f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}")
+        else:
+            write_record(binary.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} binaries[/dim]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -135,19 +152,57 @@ def list_binaries(
 # =============================================================================
 
 
-def update_binaries(version: str | None = None, abspath: str | None = None) -> int:
-    """Apply supplied fields to each JSONL-selected Binary."""
+def update_binaries(
+    version: str | None = None,
+    abspath: str | None = None,
+) -> int:
+    """
+    Update Binaries from stdin JSONL.
+
+    Reads Binary records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from archivebox.misc.jsonl import read_stdin, write_record
     from archivebox.machine.models import Binary
-    from archivebox.cli.cli_util import update_records
 
-    def update(binary):
-        if version:
-            binary.version = version
-        if abspath:
-            binary.abspath = abspath
-        binary.save()
+    is_tty = sys.stdout.isatty()
 
-    return update_records(Binary, update, plural="binaries")
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        binary_id = record.get("id")
+        if not binary_id:
+            continue
+
+        try:
+            binary = Binary.objects.get(id=binary_id)
+
+            # Apply updates from CLI flags
+            if version:
+                binary.version = version
+            if abspath:
+                binary.abspath = abspath
+
+            binary.save()
+            updated_count += 1
+
+            if not is_tty:
+                write_record(binary.to_json())
+
+        except Binary.DoesNotExist:
+            rprint(f"[yellow]Binary not found: {binary_id}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} binaries[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -156,18 +211,50 @@ def update_binaries(version: str | None = None, abspath: str | None = None) -> i
 
 
 def delete_binaries(yes: bool = False, dry_run: bool = False) -> int:
-    """Delete binaries selected by stdin JSONL; --yes confirms, --dry-run previews."""
-    from archivebox.cli.cli_util import delete_records
+    """
+    Delete Binaries from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
     from archivebox.machine.models import Binary
 
-    return delete_records(
-        Binary,
-        label="binary",
-        plural="binaries",
-        preview=lambda obj: f"{obj.name} {obj.abspath}",
-        yes=yes,
-        dry_run=dry_run,
-    )
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    binary_ids = [r.get("id") for r in records if r.get("id")]
+
+    if not binary_ids:
+        rprint("[yellow]No valid binary IDs in input[/yellow]", file=sys.stderr)
+        return 1
+
+    binaries = Binary.objects.filter(id__in=binary_ids)
+    count = binaries.count()
+
+    if count == 0:
+        rprint("[yellow]No matching binaries found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} binaries (dry run)[/yellow]", file=sys.stderr)
+        for binary in binaries:
+            rprint(f"  {binary.name} {binary.abspath}", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = binaries.delete()
+    rprint(f"[green]Deleted {deleted_count} binaries[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -185,9 +272,9 @@ def main():
 @click.option("--name", "-n", required=True, help="Binary name (e.g., chrome, wget)")
 @click.option("--abspath", "-p", required=True, help="Absolute path to binary")
 @click.option("--version", "-v", default="", help="Binary version")
-def create_cmd(**kwargs):
+def create_cmd(name: str, abspath: str, version: str):
     """Create/register a Binary."""
-    sys.exit(create_binary(**kwargs))
+    sys.exit(create_binary(name=name, abspath=abspath, version=version))
 
 
 @main.command("list")
@@ -195,25 +282,37 @@ def create_cmd(**kwargs):
 @click.option("--abspath__icontains", help="Filter by path contains")
 @click.option("--version__icontains", help="Filter by version contains")
 @click.option("--limit", type=int, help="Limit number of results")
-def list_cmd(**kwargs):
+def list_cmd(
+    name: str | None,
+    abspath__icontains: str | None,
+    version__icontains: str | None,
+    limit: int | None,
+):
     """List Binaries as JSONL."""
-    sys.exit(list_binaries(**kwargs))
+    sys.exit(
+        list_binaries(
+            name=name,
+            abspath__icontains=abspath__icontains,
+            version__icontains=version__icontains,
+            limit=limit,
+        ),
+    )
 
 
 @main.command("update")
 @click.option("--version", "-v", help="Set version")
 @click.option("--abspath", "-p", help="Set path")
-def update_cmd(**kwargs):
+def update_cmd(version: str | None, abspath: str | None):
     """Update Binaries from stdin JSONL."""
-    sys.exit(update_binaries(**kwargs))
+    sys.exit(update_binaries(version=version, abspath=abspath))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(**kwargs):
+def delete_cmd(yes: bool, dry_run: bool):
     """Delete Binaries from stdin JSONL."""
-    sys.exit(delete_binaries(**kwargs))
+    sys.exit(delete_binaries(yes=yes, dry_run=dry_run))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_crawl.py

@@ -177,20 +177,43 @@ def list_crawls(
     max_depth: int | None = None,
     limit: int | None = None,
 ) -> int:
-    """List crawls as JSONL, or formatted rows in a terminal."""
-    from archivebox.crawls.models import Crawl
-    from archivebox.cli.cli_util import list_records, format_status
+    """
+    List Crawls as JSONL with optional filters.
 
-    queryset = apply_filters(
-        Crawl.objects.order_by("-created_at"),
-        {"status": status, "urls__icontains": urls__icontains, "max_depth": max_depth},
-        limit=limit,
-    )
-    return list_records(
-        queryset,
-        plural="crawls",
-        render=lambda crawl: f"{format_status(crawl.status, 8)} [dim]{crawl.id}[/dim] {crawl.urls[:50].replace(chr(10), chr(32))}...",
-    )
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.crawls.models import Crawl
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Crawl.objects.all().order_by("-created_at")
+
+    # Apply filters
+    filter_kwargs = {
+        "status": status,
+        "urls__icontains": urls__icontains,
+        "max_depth": max_depth,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for crawl in queryset:
+        if is_tty:
+            status_color = {
+                "queued": "yellow",
+                "started": "blue",
+                "sealed": "green",
+            }.get(crawl.status, "dim")
+            url_preview = crawl.urls[:50].replace("\n", " ")
+            rprint(f"[{status_color}]{crawl.status:8}[/{status_color}] [dim]{crawl.id}[/dim] {url_preview}...")
+        else:
+            write_record(crawl.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} crawls[/dim]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -198,26 +221,73 @@ def list_crawls(
 # =============================================================================
 
 
-def update_crawls(status: str | None = None, max_depth: int | None = None) -> int:
-    """Update JSONL-selected crawls through their lifecycle operations."""
+def update_crawls(
+    status: str | None = None,
+    max_depth: int | None = None,
+) -> int:
+    """
+    Update Crawls from stdin JSONL.
+
+    Reads Crawl records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
     from django.utils import timezone
+
+    from archivebox.misc.jsonl import read_stdin, write_record
     from archivebox.crawls.models import Crawl
-    from archivebox.cli.cli_util import update_records, update_record_status
 
-    def update(crawl):
-        if status:
-            try:
-                update_record_status(crawl, status)
-            except ValueError as err:
-                rprint(f"[red]{err}[/red]", file=sys.stderr)
-                return False
-        if max_depth is not None:
-            crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
-            crawl.max_depth = max_depth
-        elif not status:
-            crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
+    is_tty = sys.stdout.isatty()
 
-    return update_records(Crawl, update, plural="crawls", refresh=True)
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        crawl_id = record.get("id")
+        if not crawl_id:
+            continue
+
+        try:
+            crawl = Crawl.objects.get(id=crawl_id)
+
+            if status:
+                if status not in Crawl.StatusChoices.values:
+                    rprint(f"[red]Invalid crawl status: {status}[/red]", file=sys.stderr)
+                    continue
+                if status == Crawl.StatusChoices.SEALED:
+                    crawl.cancel()
+                elif status == Crawl.StatusChoices.PAUSED:
+                    crawl.pause()
+                elif status == Crawl.StatusChoices.QUEUED:
+                    if crawl.status == Crawl.StatusChoices.PAUSED:
+                        crawl.resume()
+                    else:
+                        crawl.update_and_requeue(status=Crawl.StatusChoices.QUEUED, retry_at=timezone.now())
+                elif status == Crawl.StatusChoices.STARTED:
+                    crawl.update_and_requeue(status=Crawl.StatusChoices.STARTED, retry_at=timezone.now())
+            if max_depth is not None:
+                crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
+                crawl.max_depth = max_depth
+            elif not status:
+                crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
+            updated_count += 1
+
+            if not is_tty:
+                crawl.refresh_from_db()
+                write_record(crawl.to_json())
+
+        except Crawl.DoesNotExist:
+            rprint(f"[yellow]Crawl not found: {crawl_id}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} crawls[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -226,18 +296,51 @@ def update_crawls(status: str | None = None, max_depth: int | None = None) -> in
 
 
 def delete_crawls(yes: bool = False, dry_run: bool = False) -> int:
-    """Delete crawls selected by stdin JSONL; --yes confirms, --dry-run previews."""
-    from archivebox.cli.cli_util import delete_records
+    """
+    Delete Crawls from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
     from archivebox.crawls.models import Crawl
 
-    return delete_records(
-        Crawl,
-        label="crawl",
-        plural="crawls",
-        preview=lambda obj: f"[dim]{obj.id}[/dim] {obj.urls[:50].replace(chr(10), chr(32))}...",
-        yes=yes,
-        dry_run=dry_run,
-    )
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    crawl_ids = [r.get("id") for r in records if r.get("id")]
+
+    if not crawl_ids:
+        rprint("[yellow]No valid crawl IDs in input[/yellow]", file=sys.stderr)
+        return 1
+
+    crawls = Crawl.objects.filter(id__in=crawl_ids)
+    count = crawls.count()
+
+    if count == 0:
+        rprint("[yellow]No matching crawls found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} crawls (dry run)[/yellow]", file=sys.stderr)
+        for crawl in crawls:
+            url_preview = crawl.urls[:50].replace("\n", " ")
+            rprint(f"  [dim]{crawl.id}[/dim] {url_preview}...", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = crawls.delete()
+    rprint(f"[green]Deleted {deleted_count} crawls[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -266,25 +369,37 @@ def create_cmd(urls: tuple, depth: int, tag: str, status: str):
 @click.option("--urls__icontains", help="Filter by URLs contains")
 @click.option("--max-depth", type=int, help="Filter by max depth")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(**kwargs):
+def list_cmd(
+    status: str | None,
+    urls__icontains: str | None,
+    max_depth: int | None,
+    limit: int | None,
+):
     """List Crawls as JSONL."""
-    sys.exit(list_crawls(**kwargs))
+    sys.exit(
+        list_crawls(
+            status=status,
+            urls__icontains=urls__icontains,
+            max_depth=max_depth,
+            limit=limit,
+        ),
+    )
 
 
 @main.command("update")
 @click.option("--status", "-s", help="Set status")
 @click.option("--max-depth", type=int, help="Set max depth")
-def update_cmd(**kwargs):
+def update_cmd(status: str | None, max_depth: int | None):
     """Update Crawls from stdin JSONL."""
-    sys.exit(update_crawls(**kwargs))
+    sys.exit(update_crawls(status=status, max_depth=max_depth))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(**kwargs):
+def delete_cmd(yes: bool, dry_run: bool):
     """Delete Crawls from stdin JSONL."""
-    sys.exit(delete_crawls(**kwargs))
+    sys.exit(delete_crawls(yes=yes, dry_run=dry_run))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_extract.py

@@ -18,7 +18,7 @@ def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[
     ArchiveResult input is accepted as a convenient reference to its parent
     Snapshot and plugin. It is never reset or converted into queued work.
     """
-    from archivebox.misc.db import uuid_ref_query
+    from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.core.models import ArchiveResult, Snapshot
     from archivebox.misc.jsonl import TYPE_ARCHIVERESULT
 
@@ -36,7 +36,7 @@ def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[
             continue
 
         if record_type == TYPE_ARCHIVERESULT:
-            result = ArchiveResult.objects.filter(uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
+            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
             if result is not None:
                 requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
                 continue
@@ -50,7 +50,7 @@ def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[
         # convenience of passing an ArchiveResult ID by trying that reference
         # only after the Snapshot lookup misses.
         if snapshot is None and record_id and not record.get("url"):
-            result = ArchiveResult.objects.filter(uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
+            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
             if result is not None:
                 requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
                 continue
@@ -177,6 +177,11 @@ def run_plugins(
     return 0
 
 
+def process_archiveresult_by_id(archiveresult_id: str) -> int:
+    """Re-run the parent Snapshot plugin referenced by an ArchiveResult."""
+    return run_plugins((), records=[{"id": archiveresult_id}], wait=True)
+
+
 @click.command()
 @click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run")
 @click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")

archivebox/cli/archivebox_help.py

@@ -68,7 +68,7 @@ def help() -> None:
     [blue]docker compose run[/blue] [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
 
     [grey53]# using Docker:[/grey53]
-    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 8000:8000[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
+    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 5797:5797[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
 """
         if IN_DOCKER
         else ""
@@ -109,7 +109,7 @@ def help() -> None:
     [dark_green]archivebox[/dark_green] [green]add[/green] --depth=1 "https://example.com/some/page"
     [dark_green]archivebox[/dark_green] [green]list[/green] --sort=timestamp --csv=timestamp,downloaded_at,url,title
     [dark_green]archivebox[/dark_green] [green]schedule[/green] --every=day --depth=1 "https://example.com/some/feed.rss"
-    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:8000[/blue]                [grey53]# Start the Web UI / API server[/grey53]
+    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:5797[/blue]                [grey53]# Start the Web UI / API server[/grey53]
 """
         print(
             Panel(

archivebox/cli/archivebox_init.py

@@ -134,7 +134,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     print("    > Skipping orphan snapshot import during init.")
     print()
     print("    [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:")
-    print("        archivebox update")
+    print("        archivebox update --rescan --migrate-only")
 
     print("\n[green]----------------------------------------------------------------------[/green]")
 

archivebox/cli/archivebox_machine.py

@@ -25,6 +25,7 @@ __command__ = "archivebox machine"
 import sys
 
 import rich_click as click
+from rich import print as rprint
 
 from archivebox.cli.cli_util import apply_filters
 
@@ -34,21 +35,41 @@ from archivebox.cli.cli_util import apply_filters
 # =============================================================================
 
 
-def list_machines(hostname__icontains: str | None = None, os_platform: str | None = None, limit: int | None = None) -> int:
-    """List machines as JSONL, or formatted rows in a terminal."""
-    from archivebox.machine.models import Machine
-    from archivebox.cli.cli_util import list_records
+def list_machines(
+    hostname__icontains: str | None = None,
+    os_platform: str | None = None,
+    limit: int | None = None,
+) -> int:
+    """
+    List Machines as JSONL with optional filters.
 
-    queryset = apply_filters(
-        Machine.objects.order_by("-created_at"),
-        {"hostname__icontains": hostname__icontains, "os_platform": os_platform},
-        limit=limit,
-    )
-    return list_records(
-        queryset,
-        plural="machines",
-        render=lambda machine: f"[cyan]{machine.hostname:30}[/cyan] [dim]{machine.os_platform:10}[/dim] {machine.id}",
-    )
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.machine.models import Machine
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Machine.objects.all().order_by("-created_at")
+
+    # Apply filters
+    filter_kwargs = {
+        "hostname__icontains": hostname__icontains,
+        "os_platform": os_platform,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for machine in queryset:
+        if is_tty:
+            rprint(f"[cyan]{machine.hostname:30}[/cyan] [dim]{machine.os_platform:10}[/dim] {machine.id}")
+        else:
+            write_record(machine.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} machines[/dim]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -66,9 +87,15 @@ def main():
 @click.option("--hostname__icontains", help="Filter by hostname contains")
 @click.option("--os-platform", help="Filter by OS platform")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(**kwargs):
+def list_cmd(hostname__icontains: str | None, os_platform: str | None, limit: int | None):
     """List Machines as JSONL."""
-    sys.exit(list_machines(**kwargs))
+    sys.exit(
+        list_machines(
+            hostname__icontains=hostname__icontains,
+            os_platform=os_platform,
+            limit=limit,
+        ),
+    )
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_persona.py

@@ -36,10 +36,44 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_util import apply_filters, list_records
+from archivebox.cli.cli_util import apply_filters
 from archivebox.personas import importers as persona_importers
 
 
+# =============================================================================
+# Validation Helpers
+# =============================================================================
+
+
+def validate_persona_name(name: str) -> tuple[bool, str]:
+    """
+    Validate persona name to prevent path traversal attacks.
+
+    Returns:
+        (is_valid, error_message): tuple indicating if name is valid
+    """
+    if not name or not name.strip():
+        return False, "Persona name cannot be empty"
+
+    # Check for path separators
+    if "/" in name or "\\" in name:
+        return False, "Persona name cannot contain path separators (/ or \\)"
+
+    # Check for parent directory references
+    if ".." in name:
+        return False, "Persona name cannot contain parent directory references (..)"
+
+    # Check for hidden files/directories
+    if name.startswith("."):
+        return False, "Persona name cannot start with a dot (.)"
+
+    # Ensure name doesn't contain null bytes or other dangerous chars
+    if "\x00" in name or "\n" in name or "\r" in name:
+        return False, "Persona name contains invalid characters"
+
+    return True, ""
+
+
 def ensure_path_within_personas_dir(persona_path: Path) -> bool:
     """
     Verify that a persona path is within PERSONAS_DIR.
@@ -75,6 +109,7 @@ def create_personas(
     profile: str | None = None,
     source: str | None = None,
     browser_binary: str | None = None,
+    permissions: str | None = None,
 ) -> int:
     """
     Create Personas from names.
@@ -128,7 +163,10 @@ def create_personas(
             rprint(f'[red]Invalid persona name "{name}": {error_msg}[/red]', file=sys.stderr)
             continue
 
-        persona, created = Persona.objects.get_or_create(name=name)
+        persona, created = Persona.objects.get_or_create(
+            name=name,
+            defaults={"config": {"PERMISSIONS": permissions}} if permissions else {},
+        )
 
         if created:
             persona.ensure_dirs()
@@ -170,7 +208,15 @@ def create_personas(
         if created:
             rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr)
         if not is_tty:
-            write_record(persona.to_json())
+            write_record(
+                {
+                    "id": str(persona.id),
+                    "name": persona.name,
+                    "path": str(persona.path),
+                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
+                    "COOKIES_FILE": persona.COOKIES_FILE,
+                },
+            )
 
     rprint(f"[green]Created {created_count} new persona(s)[/green]", file=sys.stderr)
     return 0
@@ -192,20 +238,41 @@ def list_personas(
     Exit codes:
         0: Success (even if no results)
     """
+    from archivebox.misc.jsonl import write_record
     from archivebox.personas.models import Persona
 
-    queryset = apply_filters(
-        Persona.objects.order_by("name"),
-        {"name": name, "name__icontains": name__icontains},
-        limit=limit,
-    )
+    is_tty = sys.stdout.isatty()
 
-    def render(persona):
-        cookies = "[green]✓[/green]" if persona.COOKIES_FILE else "[dim]✗[/dim]"
-        chrome = "[green]✓[/green]" if Path(persona.CHROME_USER_DATA_DIR).exists() else "[dim]✗[/dim]"
-        return f"[cyan]{persona.name:20}[/cyan] cookies:{cookies} chrome:{chrome} [dim]{persona.path}[/dim]"
+    queryset = Persona.objects.all().order_by("name")
 
-    return list_records(queryset, plural="persona(s)", render=render)
+    # Apply filters
+    filter_kwargs = {
+        "name": name,
+        "name__icontains": name__icontains,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for persona in queryset:
+        cookies_status = "[green]✓[/green]" if persona.COOKIES_FILE else "[dim]✗[/dim]"
+        chrome_status = "[green]✓[/green]" if Path(persona.CHROME_USER_DATA_DIR).exists() else "[dim]✗[/dim]"
+
+        if is_tty:
+            rprint(f"[cyan]{persona.name:20}[/cyan] cookies:{cookies_status} chrome:{chrome_status} [dim]{persona.path}[/dim]")
+        else:
+            write_record(
+                {
+                    "id": str(persona.id),
+                    "name": persona.name,
+                    "path": str(persona.path),
+                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
+                    "COOKIES_FILE": persona.COOKIES_FILE,
+                },
+            )
+        count += 1
+
+    rprint(f"[dim]Listed {count} persona(s)[/dim]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -269,7 +336,13 @@ def update_personas(name: str | None = None) -> int:
             updated_count += 1
 
             if not is_tty:
-                write_record(persona.to_json())
+                write_record(
+                    {
+                        "id": str(persona.id),
+                        "name": persona.name,
+                        "path": str(persona.path),
+                    },
+                )
 
         except Persona.DoesNotExist:
             rprint(f"[yellow]Persona not found: {persona_id or old_name}[/yellow]", file=sys.stderr)
@@ -372,7 +445,11 @@ def main():
 
 @main.command("create")
 @click.argument("names", nargs=-1)
-@click.option("--import", "import_from", help="Import from chrome, chromium, brave, edge, a persona directory, or a live CDP URL")
+@click.option(
+    "--import",
+    "import_from",
+    help="Import from chrome, chrome-beta, chromium, brave, edge, a persona directory, or a live CDP URL",
+)
 @click.option("--source", help="Source browser user-data directory or exact profile path (including Docker mounts)")
 @click.option(
     "--browser-binary",
@@ -380,18 +457,39 @@ def main():
     help="Source Chromium browser executable; required for other Chromium-based browsers",
 )
 @click.option("--profile", help="Profile directory name under the user data dir (e.g. Default, Profile 1)")
-def create_cmd(names: tuple, import_from: str | None, profile: str | None, source: str | None, browser_binary: str | None):
+@click.option(
+    "--permissions",
+    type=click.Choice(["public", "unlisted", "private"]),
+    help="Visibility for a newly created persona (existing personas are unchanged)",
+)
+def create_cmd(
+    names: tuple,
+    import_from: str | None,
+    profile: str | None,
+    source: str | None,
+    browser_binary: str | None,
+    permissions: str | None,
+):
     """Create Personas, optionally importing from a browser profile."""
-    sys.exit(create_personas(names, import_from=import_from, profile=profile, source=source, browser_binary=browser_binary))
+    sys.exit(
+        create_personas(
+            names,
+            import_from=import_from,
+            profile=profile,
+            source=source,
+            browser_binary=browser_binary,
+            permissions=permissions,
+        ),
+    )
 
 
 @main.command("list")
 @click.option("--name", help="Filter by exact name")
 @click.option("--name__icontains", help="Filter by name contains")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(**kwargs):
+def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
     """List Personas as JSONL."""
-    sys.exit(list_personas(**kwargs))
+    sys.exit(list_personas(name=name, name__icontains=name__icontains, limit=limit))
 
 
 @main.command("open")
@@ -408,7 +506,7 @@ def open_cmd(name: str):
     from archivebox.config.common import get_config
     from archivebox.personas.models import Persona
 
-    valid, reason = persona_importers.validate_persona_name(name)
+    valid, reason = validate_persona_name(name)
     if not valid:
         raise click.ClickException(reason)
     persona = Persona.get_or_create_named(name)
@@ -474,17 +572,17 @@ def open_cmd(name: str):
 
 @main.command("update")
 @click.option("--name", "-n", help="Set new name")
-def update_cmd(**kwargs):
+def update_cmd(name: str | None):
     """Update Personas from stdin JSONL."""
-    sys.exit(update_personas(**kwargs))
+    sys.exit(update_personas(name=name))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(**kwargs):
+def delete_cmd(yes: bool, dry_run: bool):
     """Delete Personas from stdin JSONL."""
-    sys.exit(delete_personas(**kwargs))
+    sys.exit(delete_personas(yes=yes, dry_run=dry_run))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_process.py

@@ -28,6 +28,7 @@ __command__ = "archivebox process"
 import sys
 
 import rich_click as click
+from rich import print as rprint
 
 from archivebox.cli.cli_util import apply_filters
 
@@ -37,20 +38,46 @@ from archivebox.cli.cli_util import apply_filters
 # =============================================================================
 
 
-def list_processes(binary_name: str | None = None, machine_id: str | None = None, limit: int | None = None) -> int:
-    """List processes as JSONL, or exit codes and binary names in a terminal."""
+def list_processes(
+    binary_name: str | None = None,
+    machine_id: str | None = None,
+    limit: int | None = None,
+) -> int:
+    """
+    List Processes as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
     from archivebox.machine.models import Process
-    from archivebox.cli.cli_util import list_records
 
-    def render(process):
-        name = process.binary.name if process.binary else "unknown"
-        exit_code = process.exit_code if process.exit_code is not None else "?"
-        color = "green" if process.exit_code == 0 else "red" if process.exit_code else "yellow"
-        return f"[{color}]exit={exit_code:3}[/{color}] [cyan]{name:15}[/cyan] [dim]{process.id}[/dim]"
+    is_tty = sys.stdout.isatty()
 
-    queryset = Process.objects.select_related("binary", "machine").order_by("-started_at", "-created_at")
-    filters = {"binary__name": binary_name or None, "machine_id": machine_id or None}
-    return list_records(apply_filters(queryset, filters, limit=limit), plural="processes", render=render)
+    queryset = Process.objects.all().select_related("binary", "machine").order_by("-started_at", "-created_at")
+
+    # Apply filters
+    filter_kwargs = {}
+    if binary_name:
+        filter_kwargs["binary__name"] = binary_name
+    if machine_id:
+        filter_kwargs["machine_id"] = machine_id
+
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for process in queryset:
+        if is_tty:
+            binary_name_str = process.binary.name if process.binary else "unknown"
+            exit_code = process.exit_code if process.exit_code is not None else "?"
+            status_color = "green" if process.exit_code == 0 else "red" if process.exit_code else "yellow"
+            rprint(f"[{status_color}]exit={exit_code:3}[/{status_color}] [cyan]{binary_name_str:15}[/cyan] [dim]{process.id}[/dim]")
+        else:
+            write_record(process.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} processes[/dim]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -68,9 +95,15 @@ def main():
 @click.option("--binary-name", "-b", help="Filter by binary name")
 @click.option("--machine-id", "-m", help="Filter by machine ID")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(**kwargs):
+def list_cmd(binary_name: str | None, machine_id: str | None, limit: int | None):
     """List Processes as JSONL."""
-    sys.exit(list_processes(**kwargs))
+    sys.exit(
+        list_processes(
+            binary_name=binary_name,
+            machine_id=machine_id,
+            limit=limit,
+        ),
+    )
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_remove.py

@@ -4,17 +4,14 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox remove"
 
 import time
-from contextlib import nullcontext
 from pathlib import Path
 from collections.abc import Iterable
 
 import rich_click as click
-from rich.console import Console
 
 from django.db.models import QuerySet
 
 from archivebox.config import CONSTANTS
-from archivebox.config.common import get_config
 from archivebox.config.django import setup_django
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.checks import check_data_folder
@@ -23,6 +20,7 @@ from archivebox.misc.logging_util import (
     log_list_finished,
     log_removal_started,
     log_removal_finished,
+    TimedProgress,
 )
 from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
@@ -58,14 +56,16 @@ def remove(
     pattern_list = list(filter_patterns)
 
     log_list_started(pattern_list or None, filter_type)
-    progress = Console().status("Finding matching snapshots...") if get_config().SHOW_PROGRESS else nullcontext()
-    with progress:
+    timer = TimedProgress(360, prefix="      ")
+    try:
         if snapshots is None:
             snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
         # Freeze the target set up-front so a concurrent daemon writing new
         # snapshots can't extend the deletion under us, and so the cursor isn't
         # held open across the per-row deletes below.
         snapshot_pks = list(snapshots.values_list("pk", flat=True))
+    finally:
+        timer.end()
 
     if not snapshot_pks:
         log_removal_finished(0, 0)

archivebox/cli/archivebox_run.py

@@ -54,12 +54,9 @@ RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
 
 
 def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
-    # A supervised `archivebox run --daemon` is intentionally a disposable
-    # child. If it receives SIGINT/SIGTERM directly, exit with the conventional
-    # signal status so supervisord treats it as an unexpected worker death and
-    # restarts only the runner. The parent `archivebox server` owns supervisord
-    # shutdown and must not be pulled down by a killed daemon worker.
-    os._exit(128 + int(sig))
+    # A supervised daemon must exit non-zero so supervisord restarts it, while
+    # still unwinding the active crawl's hooks and persisted process state.
+    raise SystemExit(128 + sig.value)
 
 
 def process_stdin_records() -> int:
@@ -90,7 +87,7 @@ def process_stdin_records() -> int:
     )
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot, ArchiveResult
-    from archivebox.misc.db import uuid_ref_query
+    from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.crawls.models import Crawl
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Binary
@@ -117,7 +114,15 @@ def process_stdin_records() -> int:
 
         try:
             if record_type == TYPE_CRAWL:
-                crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
+                if record_id:
+                    # Existing crawl - re-queue
+                    try:
+                        crawl = Crawl.objects.get(id=record_id)
+                    except Crawl.DoesNotExist:
+                        crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
+                else:
+                    # New crawl - create it
+                    crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if crawl:
                     crawl.update_and_requeue(
@@ -130,8 +135,14 @@ def process_stdin_records() -> int:
                     queued_count += 1
 
             elif record_type == TYPE_SNAPSHOT or (record.get("url") and not record_type):
-                snapshot = Snapshot.objects.filter(id=record_id).first() if record_id else None
-                if snapshot is None:
+                if record_id:
+                    # Existing snapshot - re-queue
+                    try:
+                        snapshot = Snapshot.objects.get(id=record_id)
+                    except Snapshot.DoesNotExist:
+                        snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
+                else:
+                    # New snapshot - create it
                     snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if snapshot:
@@ -147,7 +158,7 @@ def process_stdin_records() -> int:
                 plugin_name = str(record.get("plugin") or "")
                 archiveresult = None
                 if not snapshot_id and record_id:
-                    archiveresult = ArchiveResult.objects.filter(uuid_ref_query("id", str(record_id))).select_related("snapshot").first()
+                    archiveresult = ArchiveResult.objects.filter(_uuid_ref_query("id", str(record_id))).select_related("snapshot").first()
                     if archiveresult:
                         snapshot_id = str(archiveresult.snapshot_id)
                         plugin_name = plugin_name or archiveresult.plugin
@@ -163,8 +174,12 @@ def process_stdin_records() -> int:
                     queued_count += 1
 
             elif record_type in {TYPE_BINARYREQUEST, TYPE_BINARY}:
-                binary = Binary.objects.filter(id=record_id).first() if record_id else None
-                if binary is None:
+                if record_id:
+                    try:
+                        binary = Binary.objects.get(id=record_id)
+                    except Binary.DoesNotExist:
+                        binary = Binary.from_json(record)
+                else:
                     binary = Binary.from_json(record)
 
                 if binary:
@@ -267,19 +282,26 @@ def run_runner(
         # future ownership lease cannot hide it from the unified scheduler.
         if crawl is not None:
             crawl.update_and_requeue(retry_at=now, refresh=False)
-    # Only a foreground `archivebox add` gets the interactive "abort current
-    # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
-    # orchestrators should shut down immediately and cleanly on the first signal.
-    interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
+    # Interactivity follows terminal ownership, not the spelling of the command.
+    # A supervised add worker has pipes and receives terminal intent/answers from
+    # its parent; direct run owns the terminal itself. Daemons and redirected
+    # commands must never wait for a person who cannot answer their prompt.
+    # Trust the explicit terminal-owner flag, not Process.root: when add borrows
+    # a server-owned supervisord, its worker's OS ancestry still leads to server.
+    # Gating on root=ADD silently made the first Ctrl+C abort after takeover.
+    interactive_interrupts = not daemon and (
+        (sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())) or os.environ.get("ARCHIVEBOX_INTERACTIVE_INTERRUPTS") == "1"
+    )
     if daemon:
         os.environ[RUNNER_DAEMON_ENV] = "1"
 
+    shutdown_state = None
     try:
         with (
             foreground_shutdown_signals(
                 on_signal=_exit_daemon_runner_on_signal if daemon else None,
                 raise_on_first_signal=not daemon,
-            ),
+            ) as shutdown_state,
             foreground_parent_watchdog(enabled=not daemon),
         ):
             run_pending_crawls(
@@ -288,10 +310,16 @@ def run_runner(
                 maintenance_only=maintenance_only,
                 interactive_interrupts=interactive_interrupts,
             )
+        if daemon and shutdown_state.signal_name:
+            return 128 + signal.Signals[shutdown_state.signal_name].value
         return 0
     except KeyboardInterrupt:
-        return 0
+        if daemon and shutdown_state and shutdown_state.signal_name:
+            return 128 + signal.Signals[shutdown_state.signal_name].value
+        return 130
     except asyncio.CancelledError as e:
+        if daemon and shutdown_state and shutdown_state.signal_name:
+            return 128 + signal.Signals[shutdown_state.signal_name].value
         if daemon:
             rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
             return 1
@@ -344,11 +372,11 @@ def main(
             ):
                 sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
         except KeyboardInterrupt:
-            sys.exit(0)
+            sys.exit(130)
 
     with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
         if snapshot_id:
-            sys.exit(run_snapshot_worker(snapshot_id))
+            sys.exit(run_snapshot_worker(snapshot_id, daemon=daemon))
 
         if binary_id:
             try:
@@ -368,7 +396,7 @@ def main(
         if crawl_id:
             sys.exit(
                 run_runner(
-                    daemon=False,
+                    daemon=daemon,
                     crawl_id=crawl_id,
                     maintenance_only=maintenance_only,
                 ),
@@ -383,7 +411,7 @@ def main(
             sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
 
 
-def run_snapshot_worker(snapshot_id: str) -> int:
+def run_snapshot_worker(snapshot_id: str, *, daemon: bool = False) -> int:
     from archivebox.config import CONSTANTS
     from archivebox.core.takeover_util import enter_single_runner_gate
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
@@ -398,6 +426,7 @@ def run_snapshot_worker(snapshot_id: str) -> int:
         return 0
 
     snapshot = None
+    interactive_interrupts = not daemon and sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             for _ in range(10):
@@ -406,7 +435,7 @@ def run_snapshot_worker(snapshot_id: str) -> int:
                     snapshot.update_and_requeue(retry_at=timezone.now())
                 elif snapshot.retry_at > timezone.now():
                     break
-                if not run_due_snapshot(snapshot, lock_seconds=60):
+                if not run_due_snapshot(snapshot, lock_seconds=60, interactive_interrupts=interactive_interrupts):
                     break
         return 0
     except KeyboardInterrupt:
@@ -419,7 +448,7 @@ def run_snapshot_worker(snapshot_id: str) -> int:
                 snapshot.update_and_requeue(retry_at=timezone.now())
         except Exception:
             pass
-        return 0
+        return 130
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
         import traceback

archivebox/cli/archivebox_server.py

@@ -46,13 +46,13 @@ def _split_bind_spec(spec: str) -> tuple[str, str]:
     """Split a ``host:port`` / ``host`` / ``port`` spec into ``(host, port)``.
 
     The empty strings stand in for "not provided"; the caller fills in
-    defaults. Bracketed IPv6 literals like ``[::1]:8000`` are handled.
+    defaults. Bracketed IPv6 literals like ``[::1]:5797`` are handled.
     """
     spec = (spec or "").strip()
     if not spec:
         return "", ""
     if spec.startswith("["):
-        # Bracketed IPv6: ``[::1]`` or ``[::1]:8000``
+        # Bracketed IPv6: ``[::1]`` or ``[::1]:5797``
         end = spec.find("]")
         if end == -1:
             return spec, ""  # malformed; let validator reject it
@@ -77,11 +77,11 @@ def _parse_and_validate_bind_spec(spec: str) -> tuple[str, str]:
     (normalized to ``127.0.0.1``). Bare hostnames are rejected because the
     bind address feeds Daphne, which has to listen on a numeric address;
     public hostnames belong in ``BASE_URL`` instead. Empty values fall back
-    to ``127.0.0.1`` / ``8000``.
+    to ``127.0.0.1`` / ``5797``.
     """
     raw_host, raw_port = _split_bind_spec(spec)
     host = raw_host.strip()
-    port = (raw_port or "").strip() or "8000"
+    port = (raw_port or "").strip() or "5797"
 
     if host == "" or host.lower() == "localhost":
         host = "127.0.0.1"
@@ -198,7 +198,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
             "[yellow]    routing cannot work against an IP address. Set BASE_URL explicitly, e.g.[/yellow]",
         )
         print(
-            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
+            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:5797[/yellow]",
         )
         if config.USES_SUBDOMAIN_ROUTING:
             print(
@@ -221,7 +221,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
             "[yellow]    a reverse proxy / ingress / public hostname, e.g.[/yellow]",
         )
         print(
-            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
+            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:5797[/yellow]",
         )
         print()
 
@@ -254,7 +254,7 @@ def server(
         print()
 
     # First non-empty positional arg is the bind spec; otherwise inherit from
-    # config (which defaults to "127.0.0.1:8000"). _parse_and_validate_bind_spec
+    # config (which defaults to "127.0.0.1:5797"). _parse_and_validate_bind_spec
     # hard-errors on hostnames so the rest of the server can assume a numeric
     # bind host.
     bind_spec = next((arg for arg in runserver_args if arg), "")

archivebox/cli/archivebox_snapshot.py

@@ -44,6 +44,7 @@ SNAPSHOT_LIST_CHUNK_SIZE = 5000
 
 def iter_snapshot_json(queryset: QuerySet) -> Iterator[dict[str, object]]:
     from archivebox.config import VERSION
+    from archivebox.config.common import redact_sensitive_config
     from archivebox.core.models import SnapshotTag
 
     fields = (
@@ -53,6 +54,9 @@ def iter_snapshot_json(queryset: QuerySet) -> Iterator[dict[str, object]]:
         "title",
         "bookmarked_at",
         "created_at",
+        "modified_at",
+        "config",
+        "notes",
         "timestamp",
         "depth",
         "status",
@@ -80,6 +84,9 @@ def iter_snapshot_json(queryset: QuerySet) -> Iterator[dict[str, object]]:
                 "tags": ",".join(sorted(tags_by_snapshot[row["id"]])),
                 "bookmarked_at": row["bookmarked_at"].isoformat() if row["bookmarked_at"] else None,
                 "created_at": row["created_at"].isoformat() if row["created_at"] else None,
+                "modified_at": row["modified_at"].isoformat() if row["modified_at"] else None,
+                "config": redact_sensitive_config(row["config"]),
+                "notes": row["notes"],
                 "timestamp": row["timestamp"],
                 "depth": row["depth"],
                 "status": row["status"],
@@ -144,7 +151,15 @@ def create_snapshots(
                     write_record(record)
 
                 # Input is a Crawl - get or create it, then create Snapshots for its URLs
-                crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
+                crawl = None
+                crawl_id = record.get("id")
+                if crawl_id:
+                    try:
+                        crawl = Crawl.objects.get(id=crawl_id)
+                    except Crawl.DoesNotExist:
+                        crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
+                else:
+                    crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if not crawl:
                     continue
@@ -166,7 +181,7 @@ def create_snapshots(
                         "depth": depth,
                         "status": status,
                     }
-                    snapshot = Snapshot.from_json(snapshot_record, overrides={"crawl": crawl, "created_by_id": created_by_id})
+                    snapshot = Snapshot.from_json(snapshot_record, overrides={"created_by_id": created_by_id})
                     if snapshot:
                         created_snapshots.append(snapshot)
                         if not is_tty:
@@ -360,29 +375,77 @@ def list_snapshots(
 # =============================================================================
 
 
-def update_snapshots(status: str | None = None, tag: str | None = None) -> int:
-    """Update JSONL-selected snapshots through their lifecycle operations."""
+def update_snapshots(
+    status: str | None = None,
+    tag: str | None = None,
+) -> int:
+    """
+    Update Snapshots from stdin JSONL.
+
+    Reads Snapshot records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
     from django.utils import timezone
+
+    from archivebox.misc.jsonl import read_stdin, write_record
     from archivebox.core.models import Snapshot
-    from archivebox.cli.cli_util import update_records, update_record_status
 
-    def update(snapshot):
-        if status:
-            try:
-                update_record_status(snapshot, status)
-            except ValueError as err:
-                rprint(f"[red]{err}[/red]", file=sys.stderr)
-                return False
-        if tag:
-            from archivebox.core.models import Tag
+    is_tty = sys.stdout.isatty()
 
-            tag_obj, _ = Tag.objects.get_or_create(name=tag)
-            snapshot.tags.add(tag_obj)
-            snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
-        if not status and not tag:
-            snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
 
-    return update_records(Snapshot, update, plural="snapshots", refresh=True)
+    updated_count = 0
+    for record in records:
+        snapshot_id = record.get("id")
+        if not snapshot_id:
+            continue
+
+        try:
+            snapshot = Snapshot.objects.get(id=snapshot_id)
+
+            if status:
+                if status not in Snapshot.StatusChoices.values:
+                    rprint(f"[red]Invalid snapshot status: {status}[/red]", file=sys.stderr)
+                    continue
+                if status == Snapshot.StatusChoices.SEALED:
+                    snapshot.cancel()
+                elif status == Snapshot.StatusChoices.PAUSED:
+                    snapshot.pause()
+                elif status == Snapshot.StatusChoices.QUEUED:
+                    if snapshot.status == Snapshot.StatusChoices.PAUSED:
+                        snapshot.resume()
+                    else:
+                        snapshot.update_and_requeue(status=Snapshot.StatusChoices.QUEUED, retry_at=timezone.now())
+                elif status == Snapshot.StatusChoices.STARTED:
+                    snapshot.update_and_requeue(status=Snapshot.StatusChoices.STARTED, retry_at=timezone.now())
+            if tag:
+                from archivebox.core.models import Tag
+
+                tag_obj, _ = Tag.objects.get_or_create(name=tag)
+                snapshot.tags.add(tag_obj)
+                snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
+
+            if not status and not tag:
+                snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
+            updated_count += 1
+
+            if not is_tty:
+                snapshot.refresh_from_db()
+                write_record(snapshot.to_json())
+
+        except Snapshot.DoesNotExist:
+            rprint(f"[yellow]Snapshot not found: {snapshot_id}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} snapshots[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -391,18 +454,50 @@ def update_snapshots(status: str | None = None, tag: str | None = None) -> int:
 
 
 def delete_snapshots(yes: bool = False, dry_run: bool = False) -> int:
-    """Delete snapshots selected by stdin JSONL; --yes confirms, --dry-run previews."""
-    from archivebox.cli.cli_util import delete_records
+    """
+    Delete Snapshots from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
     from archivebox.core.models import Snapshot
 
-    return delete_records(
-        Snapshot,
-        label="snapshot",
-        plural="snapshots",
-        preview=lambda obj: f"[dim]{obj.id}[/dim] {obj.url[:60]}",
-        yes=yes,
-        dry_run=dry_run,
-    )
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    snapshot_ids = [r.get("id") for r in records if r.get("id")]
+
+    if not snapshot_ids:
+        rprint("[yellow]No valid snapshot IDs in input[/yellow]", file=sys.stderr)
+        return 1
+
+    snapshots = Snapshot.objects.filter(id__in=snapshot_ids)
+    count = snapshots.count()
+
+    if count == 0:
+        rprint("[yellow]No matching snapshots found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} snapshots (dry run)[/yellow]", file=sys.stderr)
+        for snapshot in snapshots:
+            rprint(f"  [dim]{snapshot.id}[/dim] {snapshot.url[:60]}", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = snapshots.delete()
+    rprint(f"[green]Deleted {deleted_count} snapshots[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -437,17 +532,17 @@ def list_cmd(**kwargs):
 @main.command("update")
 @click.option("--status", "-s", help="Set status")
 @click.option("--tag", "-t", help="Add tag")
-def update_cmd(**kwargs):
+def update_cmd(status: str | None, tag: str | None):
     """Update Snapshots from stdin JSONL."""
-    sys.exit(update_snapshots(**kwargs))
+    sys.exit(update_snapshots(status=status, tag=tag))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(**kwargs):
+def delete_cmd(yes: bool, dry_run: bool):
     """Delete Snapshots from stdin JSONL."""
-    sys.exit(delete_snapshots(**kwargs))
+    sys.exit(delete_snapshots(yes=yes, dry_run=dry_run))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_tag.py

@@ -70,7 +70,7 @@ def create_tags(names: Iterable[str]) -> int:
         if not name:
             continue
 
-        tag, created = Tag.get_or_create_by_name(name)
+        tag, created = Tag.objects.get_or_create(name=name)
 
         if not is_tty:
             write_record(tag.to_json())
@@ -90,17 +90,42 @@ def create_tags(names: Iterable[str]) -> int:
 # =============================================================================
 
 
-def list_tags(name: str | None = None, name__icontains: str | None = None, limit: int | None = None) -> int:
-    """List tags as JSONL, or formatted rows in a terminal."""
-    from archivebox.core.models import Tag
-    from archivebox.cli.cli_util import list_records
+def list_tags(
+    name: str | None = None,
+    name__icontains: str | None = None,
+    limit: int | None = None,
+) -> int:
+    """
+    List Tags as JSONL with optional filters.
 
-    queryset = apply_filters(Tag.objects.order_by("name"), {"name": name, "name__icontains": name__icontains}, limit=limit)
-    return list_records(
-        queryset,
-        plural="tags",
-        render=lambda tag: f"[cyan]{tag.name:30}[/cyan] [dim]({tag.snapshot_set.count()} snapshots)[/dim]",
-    )
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.core.models import Tag
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Tag.objects.all().order_by("name")
+
+    # Apply filters
+    filter_kwargs = {
+        "name": name,
+        "name__icontains": name__icontains,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for tag in queryset:
+        snapshot_count = tag.snapshot_set.count()
+        if is_tty:
+            rprint(f"[cyan]{tag.name:30}[/cyan] [dim]({snapshot_count} snapshots)[/dim]")
+        else:
+            write_record(tag.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} tags[/dim]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -109,15 +134,56 @@ def list_tags(name: str | None = None, name__icontains: str | None = None, limit
 
 
 def update_tags(name: str | None = None) -> int:
-    """Apply supplied fields to each JSONL-selected Tag."""
+    """
+    Update Tags from stdin JSONL.
+
+    Reads Tag records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from archivebox.misc.jsonl import read_stdin, write_record
     from archivebox.core.models import Tag
-    from archivebox.cli.cli_util import update_records
 
-    def update(tag):
-        if name:
-            tag.rename(name)
+    is_tty = sys.stdout.isatty()
 
-    return update_records(Tag, update, plural="tags", by_name=True)
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        tag_id = record.get("id")
+        old_name = record.get("name")
+
+        if not tag_id and not old_name:
+            continue
+
+        try:
+            if tag_id:
+                tag = Tag.objects.get(id=tag_id)
+            else:
+                tag = Tag.objects.get(name=old_name)
+
+            # Apply updates from CLI flags
+            if name:
+                tag.name = name
+                tag.save()
+
+            updated_count += 1
+
+            if not is_tty:
+                write_record(tag.to_json())
+
+        except Tag.DoesNotExist:
+            rprint(f"[yellow]Tag not found: {tag_id or old_name}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} tags[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -126,19 +192,65 @@ def update_tags(name: str | None = None) -> int:
 
 
 def delete_tags(yes: bool = False, dry_run: bool = False) -> int:
-    """Delete tags selected by stdin JSONL; --yes confirms, --dry-run previews."""
-    from archivebox.cli.cli_util import delete_records
+    """
+    Delete Tags from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
     from archivebox.core.models import Tag
 
-    return delete_records(
-        Tag,
-        label="tag",
-        plural="tags",
-        preview=lambda obj: obj.name,
-        yes=yes,
-        dry_run=dry_run,
-        by_name=True,
-    )
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    # Collect tag IDs or names
+    tag_ids = []
+    tag_names = []
+    for r in records:
+        if r.get("id"):
+            tag_ids.append(r["id"])
+        elif r.get("name"):
+            tag_names.append(r["name"])
+
+    if not tag_ids and not tag_names:
+        rprint("[yellow]No valid tag IDs or names in input[/yellow]", file=sys.stderr)
+        return 1
+
+    from django.db.models import Q
+
+    query = Q()
+    if tag_ids:
+        query |= Q(id__in=tag_ids)
+    if tag_names:
+        query |= Q(name__in=tag_names)
+
+    tags = Tag.objects.filter(query)
+    count = tags.count()
+
+    if count == 0:
+        rprint("[yellow]No matching tags found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} tags (dry run)[/yellow]", file=sys.stderr)
+        for tag in tags:
+            rprint(f"  {tag.name}", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = tags.delete()
+    rprint(f"[green]Deleted {deleted_count} tags[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -163,24 +275,24 @@ def create_cmd(names: tuple):
 @click.option("--name", help="Filter by exact name")
 @click.option("--name__icontains", help="Filter by name contains")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(**kwargs):
+def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
     """List Tags as JSONL."""
-    sys.exit(list_tags(**kwargs))
+    sys.exit(list_tags(name=name, name__icontains=name__icontains, limit=limit))
 
 
 @main.command("update")
 @click.option("--name", "-n", help="Set new name")
-def update_cmd(**kwargs):
+def update_cmd(name: str | None):
     """Update Tags from stdin JSONL."""
-    sys.exit(update_tags(**kwargs))
+    sys.exit(update_tags(name=name))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(**kwargs):
+def delete_cmd(yes: bool, dry_run: bool):
     """Delete Tags from stdin JSONL."""
-    sys.exit(delete_tags(**kwargs))
+    sys.exit(delete_tags(yes=yes, dry_run=dry_run))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_update.py

@@ -7,6 +7,7 @@ import os
 import asyncio
 import shlex
 import time
+from itertools import islice
 
 from typing import TYPE_CHECKING, Any
 from collections.abc import Iterable
@@ -52,17 +53,20 @@ def _get_search_indexing_plugins() -> list[str]:
 def _build_filtered_snapshots_queryset(
     **kwargs,
 ):
-    from archivebox.core.models import Snapshot
     from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
     limit = kwargs.pop("limit", None)
+    reverse = kwargs.pop("reverse", False)
     snapshots = build_snapshot_queryset(**kwargs)
+    snapshots = snapshots.order_by(kwargs.get("sort") or "-created_at", "-id")
+    if reverse:
+        snapshots = snapshots.reverse()
     if kwargs.get("resume"):
-        snapshots = snapshots.filter(timestamp__lte=kwargs["resume"])
+        snapshots = snapshots.filter(**{"timestamp__gte" if reverse else "timestamp__lte": kwargs["resume"]})
     snapshots = snapshots.select_related("crawl")
     if limit is not None and limit > 0:
         snapshot_ids = list(snapshots.values_list("id", flat=True)[:limit])
-        snapshots = Snapshot.objects.filter(id__in=snapshot_ids).select_related("crawl")
+        snapshots = snapshots.filter(id__in=snapshot_ids)
 
     return snapshots
 
@@ -73,6 +77,7 @@ def reindex_snapshots(
     search_plugins: list[str],
     batch_size: int,
     collect_ids: bool = False,
+    reverse: bool = False,
     wait_for_turn=None,
 ) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
@@ -94,16 +99,15 @@ def reindex_snapshots(
             plugin=plugin_name,
             status__in=completed_statuses,
         )
-        candidates = snapshots.annotate(has_completed_index=Exists(completed_result)).filter(has_completed_index=False).order_by("id")
-        after_id = None
+        candidates = snapshots.annotate(has_completed_index=Exists(completed_result)).filter(has_completed_index=False)
+        candidates = candidates.order_by("created_at", "id") if reverse else candidates.order_by("-created_at", "-id")
+        remaining = candidates.select_related(None).only("id", "timestamp", "created_at").paged_iterator(chunk_size=batch_size)
         while True:
             if wait_for_turn:
                 wait_for_turn()
-            page = candidates.filter(id__gt=after_id) if after_id is not None else candidates
-            batch = list(page.select_related(None).only("id", "timestamp")[:batch_size])
+            batch = list(islice(remaining, batch_size))
             if not batch:
                 break
-            after_id = batch[-1].id
             records = [{"type": "Snapshot", "id": str(snapshot.id)} for snapshot in batch]
             stats["processed"] += len(batch)
             stats["requested"] += len(batch)
@@ -161,18 +165,18 @@ def update(
     continuous: bool = False,
     index_only: bool = False,
     migrate_only: bool = False,
+    rescan: bool = False,
+    reverse: bool = False,
     stop_daemon_stack: bool = True,
 ) -> None:
     """
-    Update snapshots: migrate old dirs, reconcile DB, and re-queue for archiving.
+    Update the database without forcing a filesystem scan by default.
 
-    Three-phase operation (without filters):
-    - Phase 1: Drain legacy archive/ directories into the current layout
-    - Phase 2: Select only stale fs_version rows through the indexed column
-    - Phase 3: Run queued snapshot-level filesystem maintenance until idle
-
-    With filters: Only phase 2 (DB query), no filesystem operations.
-    Without filters: All phases (full update).
+    --migrate-only: eagerly finish pending filesystem migrations for known rows.
+    --index-only: reconcile known snapshot assets/metadata and backfill search.
+    --rescan: discover filesystem orphans first, then repair known snapshots.
+    Explicit scans default to newest first; --reverse selects oldest first.
+    Filters select existing database snapshots for targeted updates.
     """
 
     from rich import print
@@ -229,40 +233,80 @@ def update(
                 continue
             raise SystemExit(exit_code)
 
-    filters = {
-        "filter_patterns": filter_patterns,
-        "status": status,
-        "url__icontains": url__icontains,
-        "url__istartswith": url__istartswith,
-        "tag": tag,
-        "crawl_id": crawl_id,
-        "limit": limit,
-        "sort": sort,
-        "search": search,
-        "before": before,
-        "after": after,
-    }
-    is_filtered_update = any(filters.values())
+    is_filtered_update = any(
+        (
+            filter_patterns,
+            status,
+            url__icontains,
+            url__istartswith,
+            tag,
+            crawl_id,
+            limit,
+            sort,
+            search,
+            before,
+            after,
+        ),
+    )
     touched_snapshot_ids: set[str] = set()
     exit_code = 0
 
     try:
+        if rescan and (is_filtered_update or resume or continuous):
+            raise click.UsageError("--rescan cannot be combined with filters, --resume, or --continuous; rerun it to resume automatically.")
+        if rescan:
+            migrate_only = True  # Discovery must not schedule new captures.
         wait_for_turn()
 
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             while True:
-                do_migrate = migrate_only or not index_only
-                do_index = index_only or not migrate_only
+                if rescan:
+                    from archivebox.cli.rescan import rescan_snapshots
+
+                    rescan_snapshots(
+                        wait_for_turn=wait_for_turn,
+                        reverse=reverse,
+                        scan_legacy=lambda: drain_old_archive_dirs(batch_size=batch_size, reverse=reverse),
+                    )
+                do_migrate = migrate_only or is_filtered_update and not index_only
+                do_index = index_only
+                if not do_migrate and not do_index:
+                    print(
+                        "[*] Collection ready. Filesystem migrations remain lazy; use --migrate-only, --index-only, or --rescan for explicit scans.",
+                    )
 
                 if do_migrate:
-                    if is_filtered_update:
+                    if (
+                        filter_patterns
+                        or status
+                        or url__icontains
+                        or url__istartswith
+                        or tag
+                        or crawl_id
+                        or limit
+                        or sort
+                        or search
+                        or before
+                        or after
+                    ):
                         print("[*] Processing filtered snapshots from database...")
                         stats = process_filtered_snapshots(
-                            **filters,
+                            filter_patterns=filter_patterns,
                             filter_type=filter_type,
+                            status=status,
+                            url__icontains=url__icontains,
+                            url__istartswith=url__istartswith,
+                            tag=tag,
+                            crawl_id=crawl_id,
+                            limit=limit,
+                            sort=sort,
+                            search=search,
+                            before=before,
+                            after=after,
                             resume=resume,
                             batch_size=batch_size,
-                            queue_for_archiving=True,
+                            queue_for_archiving=not migrate_only,
+                            reverse=reverse,
                             wait_for_turn=wait_for_turn,
                         )
                         print_stats(stats)
@@ -270,17 +314,13 @@ def update(
                     else:
                         stats_combined = {"phase1": {}, "phase2": {}}
 
-                        print("[*] Phase 1: Draining old archive/ directories (0.8.x → 0.9.x migration)...")
-                        stats_combined["phase1"] = drain_old_archive_dirs(
-                            resume_from=resume,
-                            batch_size=batch_size,
-                        )
-
                         print("[*] Phase 2: Selecting database snapshots with stale filesystem versions...")
                         stats_combined["phase2"] = process_all_db_snapshots(
                             batch_size=batch_size,
                             resume=resume,
                             wait_for_turn=wait_for_turn,
+                            reverse=reverse,
+                            migrate=True,
                         )
                         print_combined_stats(stats_combined)
                     # The due selectors are indexed and cheap when empty, so
@@ -288,7 +328,7 @@ def update(
                     # whole-table counts merely to decide whether to call it.
                     print("[*] Phase 3: Running filesystem maintenance until idle...")
                     if is_filtered_update:
-                        for snapshot_id in sorted(touched_snapshot_ids):
+                        for snapshot_id in sorted(touched_snapshot_ids, reverse=not reverse):
                             run_scoped_runner("--snapshot-id", snapshot_id)
                     elif migrate_only:
                         run_scoped_runner("--maintenance-only")
@@ -296,20 +336,51 @@ def update(
                         run_scoped_runner()
 
                 if do_index:
+                    from archivebox.cli.rescan import reconcile_known_snapshots
+
+                    snapshots = _build_filtered_snapshots_queryset(
+                        filter_patterns=filter_patterns,
+                        filter_type=filter_type,
+                        status=status,
+                        url__icontains=url__icontains,
+                        url__istartswith=url__istartswith,
+                        tag=tag,
+                        crawl_id=crawl_id,
+                        limit=limit,
+                        sort=sort,
+                        search=search,
+                        before=before,
+                        after=after,
+                        resume=resume,
+                        reverse=reverse,
+                    )
+                    reconcile_known_snapshots(snapshots, wait_for_turn=wait_for_turn)
                     search_plugins = _get_search_indexing_plugins()
                     if not search_plugins:
                         print("[*] No search indexing plugins are available, nothing to backfill.")
                     else:
                         snapshots = _build_filtered_snapshots_queryset(
-                            **filters,
+                            filter_patterns=filter_patterns,
                             filter_type=filter_type,
+                            status=status,
+                            url__icontains=url__icontains,
+                            url__istartswith=url__istartswith,
+                            tag=tag,
+                            crawl_id=crawl_id,
+                            limit=limit,
+                            sort=sort,
+                            search=search,
+                            before=before,
+                            after=after,
                             resume=resume,
+                            reverse=reverse,
                         )
                         stats = reindex_snapshots(
                             snapshots,
                             search_plugins=search_plugins,
                             batch_size=batch_size,
                             collect_ids=is_filtered_update,
+                            reverse=reverse,
                             wait_for_turn=wait_for_turn,
                         )
                         print_index_stats(stats)
@@ -325,21 +396,40 @@ def update(
         exit_code = 130
         exact_resume = err.__dict__.get("archivebox_resume")
         resume_cmd = ["archivebox", "update"]
+        if rescan:
+            resume_cmd.append("--rescan")
+        if reverse:
+            resume_cmd.append("--reverse")
         if migrate_only:
             resume_cmd.append("--migrate-only")
         if index_only:
             resume_cmd.append("--index-only")
-        resume_options = {
-            "batch-size": batch_size if batch_size != 500 else None,
-            "resume": exact_resume or resume,
-            "filter-type": filter_type if filter_type != "exact" else None,
-            **{key: value for key, value in filters.items() if key != "filter_patterns"},
-        }
-        for name, value in resume_options.items():
-            # Zero is a meaningful date boundary, but remains an unset limit.
-            if value or (name in {"before", "after"} and value is not None):
-                option = "crawl-id" if name == "crawl_id" else name
-                resume_cmd.extend([f"--{option}", str(value)])
+        if batch_size != 500:
+            resume_cmd.extend(["--batch-size", str(batch_size)])
+        if exact_resume or resume:
+            resume_cmd.extend(["--resume", str(exact_resume or resume)])
+        if before is not None:
+            resume_cmd.extend(["--before", str(before)])
+        if after is not None:
+            resume_cmd.extend(["--after", str(after)])
+        if filter_type != "exact":
+            resume_cmd.extend(["--filter-type", filter_type])
+        if status:
+            resume_cmd.extend(["--status", status])
+        if url__icontains:
+            resume_cmd.extend(["--url__icontains", url__icontains])
+        if url__istartswith:
+            resume_cmd.extend(["--url__istartswith", url__istartswith])
+        if tag:
+            resume_cmd.extend(["--tag", tag])
+        if crawl_id:
+            resume_cmd.extend(["--crawl-id", crawl_id])
+        if limit:
+            resume_cmd.extend(["--limit", str(limit)])
+        if sort:
+            resume_cmd.extend(["--sort", sort])
+        if search:
+            resume_cmd.extend(["--search", search])
         resume_cmd.extend(str(pattern) for pattern in filter_patterns)
         print("\n[red][X] archivebox update interrupted.[/red]")
         print("[yellow]Hint: resume this idempotent update with:[/yellow]")
@@ -357,7 +447,7 @@ def update(
             stop_own_supervisord_process()
 
 
-def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500) -> dict[str, int]:
+def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500, reverse: bool = False) -> dict[str, int]:
     """
     Drain old archive/ directories (0.8.x → 0.9.x migration).
 
@@ -423,18 +513,18 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
 
     # Scan real legacy directories only; these still contain data to migrate.
     entries = [
-        (e.stat().st_mtime, e.path)
+        (float(e.name), e.path)
         for e in all_entries
         if e.is_dir(follow_symlinks=False) and Snapshot.is_legacy_archive_dir(Path(e.path))  # Skip symlinks and 0.9.x roots
     ]
-    entries.sort(reverse=True)  # Newest first
+    entries.sort(reverse=not reverse)  # Newest first
     print(f"[*] Found {len(entries)} old directories to drain")
 
     for mtime, entry_path in entries:
         entry_path = Path(entry_path)
 
         # Resume from timestamp if specified
-        if resume_from and entry_path.name > resume_from:
+        if resume_from and (entry_path.name < resume_from if reverse else entry_path.name > resume_from):
             continue
 
         stats["processed"] += 1
@@ -536,7 +626,13 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
     return stats
 
 
-def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, wait_for_turn=None) -> dict[str, int]:
+def process_all_db_snapshots(
+    batch_size: int = 500,
+    resume: str | None = None,
+    wait_for_turn=None,
+    reverse: bool = False,
+    migrate: bool = False,
+) -> dict[str, int]:
     """Queue only snapshots whose indexed filesystem version is stale."""
     from archivebox.core.models import Snapshot
     from django.db.models import Q
@@ -553,21 +649,27 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     }
     queryset = Snapshot.objects.filter(fs_version__in=Snapshot._FS_VERSION_MIGRATION_PATHS)
     if resume:
-        queryset = queryset.filter(timestamp__lte=resume)
+        queryset = queryset.filter(**{"timestamp__gte" if reverse else "timestamp__lte": resume})
     initial_now = timezone.now()
-    rows_to_wake = queryset.filter(Q(retry_at__isnull=True) | Q(retry_at__gt=initial_now))
-    after_id = None
+    rows_to_wake = queryset if migrate else queryset.filter(Q(retry_at__isnull=True) | Q(retry_at__gt=initial_now))
+    ordered = rows_to_wake.order_by("created_at", "id") if reverse else rows_to_wake.order_by("-created_at", "-id")
+    remaining = ordered.select_related("crawl__created_by").paged_iterator(chunk_size=batch_size)
     while True:
         if wait_for_turn:
             wait_for_turn()
-        page = rows_to_wake.filter(id__gt=after_id) if after_id is not None else rows_to_wake
-        batch = list(page.only("id", "fs_version", "modified_at").order_by("id")[:batch_size])
+        batch = list(islice(remaining, batch_size))
         if not batch:
             break
-        after_id = batch[-1].id
         now = timezone.now()
         updated = 0
         for snapshot in batch:
+            if migrate:
+                from archivebox.services.runner import run_snapshot_maintenance
+
+                if wait_for_turn:
+                    wait_for_turn()
+                run_snapshot_maintenance(str(snapshot.id))
+                continue
             updated += int(
                 snapshot.safe_update(
                     {"retry_at": now, "modified_at": now},
@@ -599,6 +701,7 @@ def process_filtered_snapshots(
     resume: str | None,
     batch_size: int,
     queue_for_archiving: bool = True,
+    reverse: bool = False,
     wait_for_turn=None,
 ) -> dict[str, Any]:
     """Process snapshots matching filters (DB query only)."""
@@ -621,6 +724,7 @@ def process_filtered_snapshots(
         before=before,
         after=after,
         resume=resume,
+        reverse=reverse,
     )
 
     total = snapshots.count()
@@ -639,6 +743,10 @@ def process_filtered_snapshots(
             stats["snapshot_ids"].append(str(snapshot.id))
             update_values = {}
             updated = 0
+            if not queue_for_archiving and snapshot.fs_migration_needed:
+                from archivebox.services.runner import run_snapshot_maintenance
+
+                run_snapshot_maintenance(str(snapshot.id))
             if queue_for_archiving:
                 update_values.update(
                     {
@@ -736,6 +844,12 @@ def print_index_stats(stats: dict[str, Any]) -> None:
 @click.option("--continuous", is_flag=True, help="Run continuously as background worker")
 @click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content")
 @click.option("--migrate-only", is_flag=True, help="Only migrate filesystem and update database/index state")
+@click.option("--reverse", is_flag=True, help="Process oldest first instead of newest first for explicit scans")
+@click.option(
+    "--rescan",
+    is_flag=True,
+    help="Scan all snapshot directories for orphans and metadata repairs (slow, opt-in, safe to restart)",
+)
 @snapshot_filter_options(default_filter_type="exact")
 @docstring(update.__doc__)
 def main(**kwargs):

archivebox/cli/archivebox_version.py

@@ -47,7 +47,13 @@ def _format_binary_abspath(
                 continue
 
             relative_str = relative.as_posix()
-            return prefix + relative_str if relative_str else prefix.rstrip("/")
+            if prefix == "./":
+                return "." if not relative_str else f"./{relative_str}"
+            if prefix == "ABXPKG_LIB_DIR/":
+                return "ABXPKG_LIB_DIR" if not relative_str else f"ABXPKG_LIB_DIR/{relative_str}"
+            if prefix == "PERSONAS_DIR/":
+                return "PERSONAS_DIR" if not relative_str else f"PERSONAS_DIR/{relative_str}"
+            return "~" if not relative_str else f"~/{relative_str}"
 
     return normalized.as_posix()
 
@@ -55,26 +61,25 @@ def _format_binary_abspath(
 def _render_binary_abspath(abspath: str):
     from rich.text import Text
 
-    for prefix, color in (("ABXPKG_LIB_DIR", "bright_blue"), ("PERSONAS_DIR", "medium_purple"), ("~", "cyan"), (".", "cyan")):
-        if abspath == prefix or abspath.startswith(prefix + "/"):
-            return Text.assemble((prefix, color), (abspath.removeprefix(prefix), "green"))
+    if abspath.startswith("ABXPKG_LIB_DIR/"):
+        return Text.assemble(("ABXPKG_LIB_DIR", "bright_blue"), (abspath.removeprefix("ABXPKG_LIB_DIR"), "green"))
+    if abspath == "ABXPKG_LIB_DIR":
+        return Text("ABXPKG_LIB_DIR", style="bright_blue")
+    if abspath.startswith("PERSONAS_DIR/"):
+        return Text.assemble(("PERSONAS_DIR", "medium_purple"), (abspath.removeprefix("PERSONAS_DIR"), "green"))
+    if abspath == "PERSONAS_DIR":
+        return Text("PERSONAS_DIR", style="medium_purple")
+    if abspath.startswith("~/"):
+        return Text.assemble(("~", "cyan"), (abspath.removeprefix("~"), "green"))
+    if abspath == "~":
+        return Text("~", style="cyan")
+    if abspath.startswith("./"):
+        return Text.assemble((".", "cyan"), (abspath.removeprefix("."), "green"))
+    if abspath == ".":
+        return Text(".", style="cyan")
     return Text(abspath, style="green")
 
 
-def _binary_record_matches_runtime(installed, lib_dir: Path) -> bool:
-    if not installed or not installed.is_valid or not installed.version:
-        return False
-    try:
-        abspath = Path(installed.abspath).expanduser().resolve(strict=False)
-        if not abspath.exists():
-            return False
-        if installed.binprovider not in {"", "env", "apt", "brew"}:
-            abspath.relative_to(lib_dir)
-    except (OSError, ValueError):
-        return False
-    return True
-
-
 def _binary_row_dedupe_key(
     *,
     display_name: str,
@@ -108,7 +113,6 @@ def version(
 
     from rich.panel import Panel
     from rich.console import Console
-    from rich.live import Live
     from abx_dl.tables import binary_dependency_status, binary_dependency_table
 
     from archivebox.config import CONSTANTS
@@ -182,6 +186,9 @@ def version(
     else:
         requested_names = {name for name in (binaries or ()) if name}
 
+    def binary_is_requested(logical_name: str, actual_name: str, display_name: str) -> bool:
+        return not requested_names or bool({logical_name, actual_name, display_name} & requested_names)
+
     def plugin_may_have_requested_binary(plugin) -> bool:
         if not requested_names:
             return True
@@ -240,7 +247,7 @@ def version(
 
     prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
     failures = []
-    seen_rows: set[tuple[str, str, str, str]] = set()
+    seen_failures: set[str] = set()
 
     from archivebox.plugins.discovery import get_enabled_plugins, get_plugin_catalog
     from abx_dl.config import get_required_binary_requests
@@ -249,37 +256,29 @@ def version(
     from abxpkg.binary_service import BinaryEvent, BinaryService
 
     plugins = get_plugin_catalog()
-    enabled_plugins = plugins.select(get_enabled_plugins(config=config))
-    enabled_plugin_names = set(enabled_plugins)
-    runtime_config = normalize_runtime_config(config.for_crawl(), json_safe=False)
+    enabled_plugin_names = set(get_enabled_plugins(config=config))
+    runtime_config = normalize_runtime_config(dict(config.items()), json_safe=False)
     derived_config: dict[str, object] = {}
-    db_binaries = {}
-    db_available = False
     if in_data_dir:
         try:
             from archivebox.config.django import setup_django
 
             setup_django()
 
-            from archivebox.machine.models import Machine, Binary
+            from archivebox.machine.models import Machine
 
             machine = Machine.current()
             derived_config = normalize_runtime_config(machine.config, json_safe=False)
-            for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
-                if _binary_record_matches_runtime(binary, config.ABXPKG_LIB_DIR):
-                    db_binaries.setdefault(binary.name, binary)
-            db_available = True
 
         except Exception as e:
             prnt()
-            prnt("", f"[yellow]Warning: Could not query collection binary records; resolving through abxpkg: {e}[/yellow]")
+            prnt("", f"[yellow]Warning: Could not query collection machine config; resolving through abxpkg: {e}[/yellow]")
 
     declared_binary_specs: dict[str, dict[str, object]] = {}
-    plugin_binaries = []
     for plugin_name, plugin in plugins.items():
         if not plugin_may_have_requested_binary(plugin):
             continue
-        plugin_enabled = plugin_name in enabled_plugin_names
+        plugin_requested = plugin_name.lower() in {name.lower() for name in requested_names}
         binary_records = get_required_binary_requests(
             plugin,
             plugin.config.required_binaries,
@@ -295,11 +294,11 @@ def version(
                 else actual_name
             )
             display_name = logical_name
-            plugin_binaries.append((plugin_name, plugin_enabled, display_name, binary_record))
-            if not plugin_enabled and not requested_names:
-                continue
-            if _binary_record_matches_runtime(db_binaries.get(logical_name), config.ABXPKG_LIB_DIR):
+            if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
                 continue
+            # This is a diagnostic: recheck the provider instead of trusting an
+            # installation record or a cached version, including disabled plugins.
+            binary_record["no_cache"] = True
             signature = json.dumps(binary_record, sort_keys=True, default=str)
             declared_binary_specs.setdefault(signature, binary_record)
 
@@ -317,82 +316,121 @@ def version(
 
         loaded_binaries = asyncio.run(resolve_declared_binaries())
 
-    rows: list[dict[str, object]] = []
+    rows_by_binary: dict[str, dict] = {}
+    any_rows = False
     any_available = False
     compact_paths = console.is_terminal
-    live_cm = Live(binary_dependency_table(rows), console=console, refresh_per_second=8) if console.is_terminal else None
-
-    def emit_binary(plugin_name, plugin_enabled, display_name, binary, *, dim=False):
-        nonlocal any_available
-        valid = binary is not None
-        abspath = binary.abspath if valid else ""
-        version_str = str(binary.version or "unknown")[:15] if valid else "-"
-        provider = str(binary.binprovider or "env")[:8] if valid else "-"
-        any_available |= valid
-        if not valid and plugin_enabled and display_name not in failures:
-            failures.append(display_name)
-        row_key = _binary_row_dedupe_key(
-            display_name=display_name,
-            valid=valid,
-            version=version_str,
-            provider=provider,
-            abspath=abspath,
+    for plugin_name, plugin in plugins.items():
+        if not plugin_may_have_requested_binary(plugin):
+            continue
+        plugin_requested = plugin_name.lower() in {name.lower() for name in requested_names}
+        plugin_enabled = plugin_name in enabled_plugin_names
+        disabled_by = ", ".join(
+            f"{dependency.enabled_key}=False"
+            for dependency in plugins.select([plugin_name]).values()
+            if not config.get(dependency.enabled_key, True)
         )
-        if row_key in seen_rows:
-            return
-        seen_rows.add(row_key)
-        rendered_path = "[grey53]not installed[/grey53]"
-        if valid:
-            display_path = (
-                _format_binary_abspath(
-                    abspath,
-                    pwd=Path.cwd(),
-                    lib_dir=config.ABXPKG_LIB_DIR,
-                    personas_dir=CONSTANTS.PERSONAS_DIR,
-                    home=Path.home(),
-                )
-                if compact_paths
-                else abspath
+        binary_records = get_required_binary_requests(
+            plugin,
+            plugin.config.required_binaries,
+            overrides=runtime_config,
+            derived_overrides=derived_config,
+            run_output_dir=CONSTANTS.DATA_DIR,
+        )
+        for binary_record in binary_records:
+            binary_record["no_cache"] = True
+            actual_name = str(binary_record["name"])
+            logical_name = (
+                Path(actual_name).expanduser().name
+                if ("/" in actual_name or "\\" in actual_name or actual_name.startswith("~"))
+                else actual_name
             )
-            rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
-        rows.append(
-            {
+            display_name = logical_name
+            if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
+                continue
+
+            loaded = loaded_binaries[json.dumps(binary_record, sort_keys=True, default=str)]
+            abspath = loaded.abspath if loaded is not None else ""
+            version_str = str(loaded.version or "unknown") if loaded is not None else "unknown"
+            provider = str(loaded.binprovider or "env") if loaded is not None else "env"
+            valid = loaded is not None and Path(abspath).is_file()
+
+            any_rows = True
+            if valid:
+                display_path = (
+                    _format_binary_abspath(
+                        abspath,
+                        pwd=Path.cwd(),
+                        lib_dir=config.ABXPKG_LIB_DIR,
+                        personas_dir=CONSTANTS.PERSONAS_DIR,
+                        home=Path.home(),
+                    )
+                    if compact_paths
+                    else abspath
+                )
+                rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
+                any_available = True
+            else:
+                rendered_path = "[grey53]not installed[/grey53]"
+                if plugin_enabled and display_name not in seen_failures:
+                    failures.append(display_name)
+                    seen_failures.add(display_name)
+
+            if not plugin_enabled:
+                rendered_path = f"[grey53]disabled by {disabled_by or 'PLUGINS configuration'}[/grey53]"
+
+            row = {
                 "plugin": plugin_name,
                 "status": binary_dependency_status(enabled=plugin_enabled, valid=valid),
                 "binary": display_name,
-                "version": version_str,
-                "provider": provider,
+                "version": version_str if valid else "-",
+                "provider": provider if valid else "-",
                 "path": rendered_path,
-                "style": "dim" if dim or not plugin_enabled else "",
-            },
-        )
-        if live_cm is not None:
-            live_cm.update(binary_dependency_table(rows), refresh=True)
+                "style": "" if plugin_enabled else "dim",
+                "enabled": plugin_enabled,
+                "valid": valid,
+                "plugins": [plugin_name],
+                "disabled_by": [disabled_by or "PLUGINS configuration"] if not plugin_enabled else [],
+            }
+            existing = rows_by_binary.get(display_name)
+            if existing is None:
+                rows_by_binary[display_name] = row
+            else:
+                consumers = list(dict.fromkeys([*existing["plugins"], plugin_name]))
+                disabled_reasons = list(dict.fromkeys([*existing["disabled_by"], *row["disabled_by"]]))
+                # Disabled consumers cannot mask an active requirement. If any
+                # active request fails (e.g. a version floor), keep that failure.
+                if plugin_enabled and (not existing["enabled"] or not valid):
+                    existing.update(row)
+                existing["plugins"] = consumers
+                existing["disabled_by"] = disabled_reasons
+                existing["plugin"] = ", ".join(consumers)
+                if not existing["enabled"]:
+                    existing["path"] = f"[grey53]disabled by {', '.join(disabled_reasons)}[/grey53]"
 
-    if live_cm is not None:
-        live_cm.start()
-    try:
-        for plugin_name, plugin_enabled, display_name, binary_record in plugin_binaries:
-            installed = db_binaries.get(display_name) if db_available else None
-            if not _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
-                # Resolve only enabled or explicitly requested dependencies. Cold-loading
-                # every optional provider makes status and migration checks needlessly slow.
-                if not plugin_enabled and not requested_names:
-                    continue
-                installed = loaded_binaries[json.dumps(binary_record, sort_keys=True, default=str)]
-            emit_binary(plugin_name, plugin_enabled, display_name, installed)
+    rows = list(rows_by_binary.values())
 
-        if db_available:
-            for binary_name, installed in db_binaries.items():
-                if not requested_names or binary_name in requested_names:
-                    emit_binary("(database)", True, binary_name, installed, dim=True)
-    finally:
-        if live_cm is not None:
-            live_cm.stop()
-        else:
-            prnt(binary_dependency_table(rows))
+    if console.is_terminal and console.width < 120:
+        from rich.text import Text
 
-    if not rows:
+        for row in rows:
+            heading = Text.from_markup(str(row["status"]))
+            heading.append(f" {row['binary']} {row['version']} ({row['provider']}) · {row['plugin']}")
+            prnt(heading, style=str(row["style"]))
+            path = row["path"]
+            prnt(Text.assemble("   ", path if isinstance(path, Text) else Text.from_markup(str(path))), style=str(row["style"]))
+    else:
+        table = binary_dependency_table(rows)
+        table.expand = False
+        table.title = None
+        table.box = None
+        for column in table.columns:
+            column.width = None
+            column.max_width = None
+            column.ratio = None
+        prnt(table)
+
+    if not any_rows:
         prnt("", "[grey53]No required binaries declared for discovered plugins.[/grey53]")
     elif not any_available:
         prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")

archivebox/cli/cli_util.py

@@ -42,139 +42,3 @@ def apply_filters(queryset, filter_kwargs: dict, limit: int | None = None):
         queryset = queryset[:limit]
 
     return queryset
-
-
-def delete_records(
-    model,
-    *,
-    label: str,
-    plural: str,
-    preview,
-    yes: bool,
-    dry_run: bool,
-    preview_limit: int | None = None,
-    by_name: bool = False,
-) -> int:
-    """Delete a JSONL selection with one confirmation and dry-run contract.
-
-    Keep queryset deletion so Django's cascades and model cleanup signals run.
-    Persona deletion has additional filesystem policy and stays with that command.
-    """
-    import sys
-
-    from django.db.models import Q
-    from rich import print as rprint
-
-    from archivebox.misc.jsonl import read_stdin
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-    ids = [record["id"] for record in records if record.get("id")]
-    names = [record["name"] for record in records if by_name and not record.get("id") and record.get("name")]
-    if not ids and not names:
-        rprint(f"[yellow]No valid {label} IDs{' or names' if by_name else ''} in input[/yellow]", file=sys.stderr)
-        return 1
-    query = Q(id__in=ids)
-    if names:
-        query |= Q(name__in=names)
-    queryset = model.objects.filter(query)
-    count = queryset.count()
-    if not count:
-        rprint(f"[yellow]No matching {plural} found[/yellow]", file=sys.stderr)
-        return 0
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} {plural} (dry run)[/yellow]", file=sys.stderr)
-        for obj in queryset[:preview_limit] if preview_limit is not None else queryset:
-            rprint(f"  {preview(obj)}", file=sys.stderr)
-        if preview_limit is not None and count > preview_limit:
-            rprint(f"  ... and {count - preview_limit} more", file=sys.stderr)
-        return 0
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-    deleted_count, _ = queryset.delete()
-    rprint(f"[green]Deleted {deleted_count} {plural}[/green]", file=sys.stderr)
-    return 0
-
-
-def update_record_status(record, status: str) -> None:
-    """Route CLI status changes through the model's lifecycle operations."""
-    from django.utils import timezone
-
-    if status not in record.StatusChoices.values:
-        raise ValueError(f"Invalid {record._meta.model_name} status: {status}")
-    transitions = {"sealed": record.cancel, "paused": record.pause}
-    if status == "queued" and record.is_paused:
-        record.resume()
-    elif transition := transitions.get(status):
-        transition()
-    else:
-        record.update_and_requeue(status=status, retry_at=timezone.now())
-
-
-def list_records(queryset, *, plural: str, render) -> int:
-    """Stream JSONL when piped; render rows and a count in a terminal."""
-    import sys
-    from rich import print as rprint
-    from archivebox.misc.jsonl import write_record
-
-    is_tty = sys.stdout.isatty()
-    count = 0
-    for record in queryset:
-        if is_tty:
-            rprint(render(record))
-        else:
-            write_record(record.to_json())
-        count += 1
-    rprint(f"[dim]Listed {count} {plural}[/dim]", file=sys.stderr)
-    return 0
-
-
-def format_status(status: str, width: int) -> str:
-    colors = {
-        "queued": "yellow",
-        "started": "blue",
-        "sealed": "green",
-        "succeeded": "green",
-        "failed": "red",
-        "backoff": "magenta",
-    }
-    color = colors.get(status, "dim")
-    return f"[{color}]{status:{width}}[/{color}]"
-
-
-def update_records(model, update, *, plural: str, by_name: bool = False, refresh: bool = False) -> int:
-    """Apply a command's PATCH operation to each JSONL-selected model.
-
-    Returning False from the operation skips that row. Model operations stay in
-    their commands so lifecycle changes and ordinary field saves remain explicit.
-    """
-    import sys
-    from rich import print as rprint
-    from archivebox.misc.jsonl import read_stdin, write_record
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-    is_tty = sys.stdout.isatty()
-    count = 0
-    for record in records:
-        key = "id" if record.get("id") else "name" if by_name and record.get("name") else None
-        if key is None:
-            continue
-        try:
-            obj = model.objects.get(**{key: record[key]})
-            if update(obj) is False:
-                continue
-            count += 1
-            if not is_tty:
-                if refresh:
-                    obj.refresh_from_db()
-                write_record(obj.to_json())
-        except model.DoesNotExist:
-            rprint(f"[yellow]{model.__name__} not found: {record[key]}[/yellow]", file=sys.stderr)
-    rprint(f"[green]Updated {count} {plural}[/green]", file=sys.stderr)
-    return 0

archivebox/cli/rescan.py

@@ -0,0 +1,222 @@
+"""Explicit discovery of portable snapshots; no checkpoint or default tree walk."""
+
+import json
+import os
+import uuid
+from itertools import groupby
+from pathlib import Path
+
+from django.contrib.auth import get_user_model
+from django.core.exceptions import ValidationError
+from django.db import IntegrityError
+from rich import print
+
+from archivebox.config import CONSTANTS
+from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.crawls.models import Crawl
+from archivebox.misc.util import parse_date
+
+
+def _directories(parent: Path):
+    # Names encode dates / UUIDv7 creation time. Newest first within each
+    # subtree, with bounded memory and no traversal into plugin payloads.
+    with os.scandir(parent) as entries:
+        paths = [Path(entry.path) for entry in entries if entry.is_dir(follow_symlinks=False)]
+    yield from sorted(paths, key=lambda path: path.name, reverse=True)
+
+
+def _snapshot_directories(reverse=False):
+    dates = []
+    for user in _directories(CONSTANTS.USERS_DIR):
+        root = user / CONSTANTS.SNAPSHOTS_DIR_NAME
+        if root.is_symlink() or not root.is_dir():
+            continue
+        dates.extend(_directories(root))
+    dates.sort(key=lambda path: path.name, reverse=not reverse)
+    for _, date_group in groupby(dates, key=lambda path: path.name):
+        directories = [directory for date in date_group for domain in _directories(date) for directory in _directories(domain)]
+        yield from sorted(directories, key=lambda path: path.name, reverse=not reverse)
+
+
+def _fields(record, names):
+    return {name: record[name] for name in names.split() if name in record}
+
+
+def _dates(record):
+    return {name: parse_date(record[name]) for name in ("created_at", "modified_at") if record.get(name)}
+
+
+def _restore_dates(obj, record):
+    dates = _dates(record)
+    if dates:
+        type(obj).objects.filter(pk=obj.pk).update(**dates)
+        for name, value in dates.items():
+            setattr(obj, name, value)
+
+
+def _reconcile_directory(directory: Path) -> str:
+    snapshot_id = uuid.UUID(directory.name)
+    snapshot = Snapshot.objects.select_related("crawl__created_by").filter(pk=snapshot_id).first()
+    index = directory / CONSTANTS.JSONL_INDEX_FILENAME
+    if index.is_symlink():
+        raise ValueError("metadata must not be a symlink")
+    if not index.exists():
+        if snapshot is None or Path(snapshot.output_dir) != directory or snapshot.status != "sealed":
+            raise ValueError("missing index.jsonl; cannot reconstruct an unknown or active snapshot")
+        snapshot.write_index_jsonl(output_dir=directory)
+        snapshot.reconcile_filesystem_links()
+        return "Repaired"
+
+    records = [json.loads(line) for line in index.read_text().splitlines() if line.strip()]
+    if any(not isinstance(record, dict) for record in records):
+        raise ValueError("metadata records must be JSON objects")
+    snapshots = [record for record in records if record.get("type") == "Snapshot"]
+    if len(snapshots) != 1:
+        raise ValueError("expected exactly one Snapshot record")
+    record = snapshots[0]
+    if uuid.UUID(record["id"]) != snapshot_id:
+        raise ValueError("snapshot ID conflicts with directory name")
+    if record.get("status") != "sealed":
+        raise ValueError("snapshot is not sealed; finish the source capture first")
+    if record.get("fs_version") not in {Snapshot._fs_current_version(), *Snapshot._FS_VERSION_MIGRATION_PATHS}:
+        raise ValueError("unsupported filesystem version")
+    crawl_id = uuid.UUID(record["crawl_id"])
+    owner = directory.parents[3].name
+    crawl_records = [item for item in records if item.get("type") == "Crawl" and uuid.UUID(item["id"]) == crawl_id]
+    crawl_record = crawl_records[0] if crawl_records else {}
+    if crawl_record.get("created_by", owner) != owner:
+        raise ValueError("crawl owner conflicts with directory path")
+    crawl = Crawl.objects.select_related("created_by").filter(pk=crawl_id).first()
+    imported_crawl = crawl is None
+    if crawl and crawl.created_by.username != owner:
+        raise ValueError("existing crawl ID belongs to a different owner")
+    if snapshot and (snapshot.url != record["url"] or snapshot.crawl_id != crawl_id or snapshot.timestamp != record["timestamp"]):
+        raise ValueError("existing snapshot ID has conflicting identity; refusing to overwrite")
+    if snapshot and Path(snapshot.output_dir) != directory:
+        raise ValueError("existing snapshot has a different storage path")
+
+    # Validate identities before inserting anything. Never turn an ID collision
+    # into a new ID, and never overwrite a newer destination row.
+    # write_index_jsonl retains historical hook records before appending the
+    # canonical DB rows. The final record for each hook is authoritative.
+    results = {(item.get("plugin"), item.get("hook_name", "")): item for item in records if item.get("type") == "ArchiveResult"}.values()
+    pending = []
+    for result in results:
+        result_id = uuid.UUID(result["id"])
+        if uuid.UUID(result["snapshot_id"]) != snapshot_id:
+            raise ValueError("ArchiveResult belongs to another snapshot")
+        existing = ArchiveResult.objects.filter(pk=result_id).first()
+        identity = (snapshot_id, result["plugin"], result.get("hook_name", ""))
+        if existing:
+            if (existing.snapshot_id, existing.plugin, existing.hook_name) != identity:
+                raise ValueError("existing ArchiveResult ID has conflicting identity")
+        elif ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin=identity[1], hook_name=identity[2]).exists():
+            raise ValueError("existing hook has a different ArchiveResult ID")
+        else:
+            pending.append(result)
+
+    imported = snapshot is None
+    if imported:
+        if Snapshot.objects.filter(timestamp=record["timestamp"]).exists():
+            raise ValueError("timestamp belongs to another snapshot")
+        if Snapshot.objects.filter(crawl_id=crawl_id, url=record["url"]).exists():
+            raise ValueError("crawl URL belongs to another snapshot")
+        user, _ = get_user_model().objects.get_or_create(
+            username=owner,
+            defaults={"is_active": False, "password": "!"},
+        )
+        if crawl is None:
+            # Old exports contain only crawl_id. Keep that identity but report
+            # that absent crawl metadata cannot be reconstructed exactly.
+            if not crawl_record:
+                print(f"[yellow]    {directory}: no Crawl record; preserving ID with recovered metadata[/yellow]")
+            crawl = Crawl(
+                id=crawl_id,
+                created_by=user,
+                status="sealed",
+                retry_at=None,
+                **_fields(crawl_record, "urls max_depth config tags_str label"),
+            )
+            crawl.save(force_insert=True)
+            _restore_dates(crawl, crawl_record)
+        snapshot = Snapshot.create_from_directory(directory, crawl=crawl, preserve_identity=True)
+        if snapshot is None:
+            raise ValueError("invalid snapshot metadata")
+        snapshot.status = "sealed"
+        snapshot.retry_at = None
+        if "config" not in record:
+            snapshot.config = {"PERMISSIONS": "private"}
+        if Path(snapshot.get_storage_path_for_version(Snapshot._fs_current_version())) != directory:
+            raise ValueError("metadata does not match its owner/date/domain directory")
+        snapshot.save(force_insert=True)
+        _restore_dates(snapshot, record)
+
+    # Each insert is durable. An interruption after the Snapshot insert is not
+    # completion: the next pass still imports missing result IDs and tags.
+    for result in pending:
+        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=True)
+    tags_before = set(snapshot.tags.values_list("name", flat=True))
+    snapshot._merge_tags_from_index(record)
+    tags_changed = tags_before != set(snapshot.tags.values_list("name", flat=True))
+    link = Path(snapshot.crawl.output_dir) / "snapshots" / snapshot.extract_domain_from_url(snapshot.url) / str(snapshot.id)
+    links_ok = link.is_symlink() and link.resolve() == directory.resolve()
+    if not links_ok:
+        snapshot.reconcile_filesystem_links()
+    if snapshot.fs_migration_needed:
+        snapshot.migrate_filesystem_to_current_version(source_dir=directory)
+    if imported:
+        _restore_dates(snapshot, record)
+    if imported_crawl:
+        _restore_dates(crawl, crawl_record)
+    # Leave source metadata untouched, including process/binary records that
+    # are historical evidence rather than runnable destination process state.
+    return "Imported" if imported else "Repaired" if pending or tags_changed or not links_ok else "Unchanged"
+
+
+def reconcile_known_snapshots(snapshots, wait_for_turn=None):
+    """Explicit metadata / asset repair; normal updates never call this scan."""
+    for snapshot in snapshots.filter(status="sealed").paged_iterator():
+        if wait_for_turn:
+            wait_for_turn()
+        if snapshot.fs_migration_needed:
+            snapshot.migrate_filesystem_to_current_version()
+        directory = Path(snapshot.output_dir)
+        if not directory.is_dir():
+            raise ValueError(f"Missing snapshot directory: {directory}")
+        _reconcile_directory(directory)
+        for result in snapshot.archiveresult_set.all():
+            result.update_output_metadata_from_filesystem(snapshot_dir=directory, full_scan=True)
+
+
+def rescan_snapshots(wait_for_turn=None, reverse=False, scan_legacy=None):
+    stats = dict.fromkeys(("Imported", "Repaired", "Unchanged", "Errors"), 0)
+    print("[*] Rescanning snapshot directories (O(N), orphans first; completed records are reused)...")
+    known = {str(pk) for pk in Snapshot.objects.values_list("pk", flat=True)}
+    # Two streaming passes avoid enumerating the entire archive before the
+    # first import. Rows imported before interruption move to the second pass
+    # next time; only one day's directory names are held in memory at a time.
+    for existing in (False, True):
+        if existing and scan_legacy:
+            scan_legacy()
+        for directory in _snapshot_directories(reverse=reverse):
+            if (directory.name.replace("-", "") in known) != existing:
+                continue
+            if wait_for_turn:
+                wait_for_turn()
+            try:
+                outcome = _reconcile_directory(directory)
+                if directory.name.replace("-", "") in known:
+                    for result in ArchiveResult.objects.filter(snapshot_id=uuid.UUID(directory.name)):
+                        result.update_output_metadata_from_filesystem(snapshot_dir=directory, full_scan=True)
+                stats[outcome] += 1
+                if outcome != "Unchanged":
+                    print(f"    {outcome}: {directory}")
+            except (OSError, ValueError, KeyError, TypeError, ValidationError, IntegrityError) as err:
+                stats["Errors"] += 1
+                print(f"[yellow]    Unresolved: {directory}: {err}[/yellow]")
+            if sum(stats.values()) % 100 == 0:
+                print("    " + ", ".join(f"{key}: {value}" for key, value in stats.items()))
+    print("    " + ", ".join(f"{key}: {value}" for key, value in stats.items()))
+    if stats["Errors"]:
+        raise SystemExit(1)
+    return stats

archivebox/config/common.py

@@ -83,12 +83,16 @@ def base_url_from_legacy_server_config(raw_config: Mapping[str, object]) -> str:
             return candidate
 
     listen_host = str(raw_config.get("LISTEN_HOST") or "").strip()
-    if listen_host and listen_host.lower() != "archivebox.localhost:8000":
+    if listen_host and listen_host.lower() not in {"archivebox.localhost:8000", "archivebox.localhost:5797"}:
         return normalize(listen_host)
 
     csrf_origins = [normalize(origin) for origin in str(raw_config.get("CSRF_TRUSTED_ORIGINS") or "").split(",")]
     csrf_origins = [origin for origin in csrf_origins if origin]
-    nondefault_csrf_origins = [origin for origin in csrf_origins if origin != "http://admin.archivebox.localhost:8000"]
+    nondefault_csrf_origins = [
+        origin
+        for origin in csrf_origins
+        if origin not in {"http://admin.archivebox.localhost:8000", "http://admin.archivebox.localhost:5797"}
+    ]
     if len(nondefault_csrf_origins) == 1:
         return nondefault_csrf_origins[0]
 
@@ -314,7 +318,7 @@ class ServerConfig(BaseConfigSet):
     )
 
     SECRET_KEY: str = Field(default_factory=lambda: "".join(secrets.choice("abcdefghijklmnopqrstuvwxyz0123456789_") for _ in range(50)))
-    BIND_ADDR: str = Field(default="127.0.0.1:8000")
+    BIND_ADDR: str = Field(default="127.0.0.1:5797")
     BASE_URL: str = Field(default="")
     ALLOWED_HOSTS: str = Field(default="*")
     CSRF_TRUSTED_ORIGINS: str = Field(default="")
@@ -464,7 +468,13 @@ class ArchivingConfig(BaseConfigSet):
     COOKIES_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     AUTH_STORAGE_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
-    URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST")
+    URL_DENYLIST: str = Field(
+        default=(
+            r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$"
+            r"|^https?://news\.ycombinator\.com/(?:vote|hide)/?(?:[?#]|$)"
+        ),
+        alias="URL_BLACKLIST",
+    )
     URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST")
 
     DEFAULT_PERSONA: str = Field(default="Default", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})

archivebox/core/admin_archiveresults.py

@@ -1,5 +1,7 @@
 __package__ = "archivebox.core"
 
+import html
+import json
 import os
 import shlex
 from functools import reduce
@@ -14,16 +16,21 @@ from django.core.exceptions import PermissionDenied, SuspiciousOperation, Valida
 from django.db.models import Count, Min, Prefetch, Q, Subquery, TextField, Window
 from django.db.models.functions import Cast
 from django.shortcuts import redirect
-from django.template.loader import render_to_string
-from django.urls import reverse
+from django.urls import resolve, reverse
+from django.utils import timezone
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django.utils.text import smart_split
 
-from archivebox.base_models.admin import card_fieldset, BaseModelAdmin
+from archivebox.base_models.admin import BaseModelAdmin
 from archivebox.core.models import ArchiveResult, Snapshot
-from archivebox.core.routes_util import build_snapshot_url
-from archivebox.core.widgets import InlineTagEditorWidget, render_copy_block
+from archivebox.core.routes_util import (
+    build_snapshot_detail_url,
+    build_snapshot_files_url,
+    build_snapshot_url,
+    build_snapshot_zip_url,
+)
+from archivebox.core.widgets import InlineTagEditorWidget
 from archivebox.machine.env_util import env_to_shell_exports
 from archivebox.misc.logging_util import printable_filesize
 from archivebox.misc.paginators import AcceleratedPaginator
@@ -114,58 +121,302 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None, can_del
     }
 
     rows = []
+    delete_url = html.escape(reverse("admin:core_archiveresult_changelist"), quote=True)
     for idx, result in enumerate(results):
         status = result.status or "queued"
         color, bg = status_colors.get(status, ("#6b7280", "#f3f4f6"))
+        output_files = result.output_files or {}
+        if isinstance(output_files, (dict, list, tuple, set)):
+            output_file_count = len(output_files)
+        elif isinstance(output_files, str):
+            try:
+                parsed = json.loads(output_files)
+                output_file_count = len(parsed) if isinstance(parsed, (dict, list, tuple, set)) else 0
+            except (TypeError, ValueError):
+                output_file_count = 0
+        else:
+            output_file_count = 0
+        output_size = int(result.output_size or 0)
+        output_size_display = html.escape(printable_filesize(output_size))
+
+        # Get plugin icon
+        icon = get_plugin_icon(result.plugin)
+
+        # Keep each timestamp component intact while allowing one wrap between them.
+        if result.end_ts:
+            end_time = (
+                f'<span style="white-space: nowrap;">{result.end_ts:%Y-%m-%d}</span>'
+                f'<wbr> <span style="white-space: nowrap;">{result.end_ts:%H:%M:%S}</span>'
+            )
+        else:
+            end_time = "-"
+
         process = result.process_record
-        output = result.output_str_for_display() or "-"
+        process_display = "-"
+        if process:
+            process_url = html.escape(reverse("admin:machine_process_change", args=[process.id]), quote=True)
+            process_label = html.escape(get_process_link_label(process), quote=True)
+            process_display = f'''
+                <a href="{process_url}"
+                   style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
+                   title="View process">{process_label}</a>
+            '''
+
+        machine_display = "-"
+        if process and process.machine_id:
+            machine_url = html.escape(reverse("admin:machine_machine_change", args=[process.machine_id]), quote=True)
+            machine_label = html.escape(str(process.machine.hostname or ""), quote=True)
+            machine_display = f'''
+                <a href="{machine_url}"
+                   style="color: #2563eb; text-decoration: none; font-size: 12px;"
+                   title="View machine">{machine_label}</a>
+            '''
+
+        # Truncate output for display
+        full_output_raw = result.output_str_for_display() or "-"
+        output_display_raw = full_output_raw[:60]
+        if len(full_output_raw) > 60:
+            output_display_raw += "..."
+        full_output = html.escape(full_output_raw)
+        output_display = html.escape(output_display_raw)
+
+        display_cmd = build_abx_dl_display_command(result)
+        replay_cmd = build_abx_dl_replay_command(result, config=config)
+        cmd_str_escaped = html.escape(display_cmd)
+        cmd_attr = html.escape(replay_cmd, quote=True)
+
+        # Build output link - use embed_path() which checks output_files first
         embed_path = result.embed_path()
-        rows.append(
-            {
-                "result": result,
-                "status": status,
-                "color": color,
-                "bg": bg,
-                "icon": get_plugin_icon(result.plugin),
-                "file_count": result.output_file_stats()[0],
-                "output_size": int(result.output_size or 0),
-                "output_size_display": printable_filesize(int(result.output_size or 0)),
-                "end_date": result.end_ts.strftime("%Y-%m-%d") if result.end_ts else "",
-                "end_time": result.end_ts.strftime("%H:%M:%S") if result.end_ts else "",
-                "process": process,
-                "process_url": reverse("admin:machine_process_change", args=[process.id]) if process else "",
-                "process_label": get_process_link_label(process) if process else "",
-                "machine_url": reverse("admin:machine_machine_change", args=[process.machine_id]) if process and process.machine_id else "",
-                "output": output,
-                "output_preview": output[:60] + ("..." if len(output) > 60 else ""),
-                "display_command": build_abx_dl_display_command(result),
-                "replay_command": build_abx_dl_replay_command(result, config=config),
-                "output_url": build_snapshot_url(
-                    str(result.snapshot_id),
-                    embed_path if embed_path and status == "succeeded" else "",
-                    config=config,
-                ),
-                "change_url": reverse("admin:core_archiveresult_change", args=[result.id]),
-                "version": result.cmd_version or "-",
-                "pwd": str(result.pwd or "-"),
-                "dom_id": f"output_{idx}_{str(result.id)[:8]}",
-                "short_id": str(result.id)[-8:],
-            },
+        snapshot_id = str(result.snapshot_id)
+        if embed_path and result.status == "succeeded":
+            output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
+        else:
+            output_link = build_snapshot_detail_url(result.snapshot.archive_path_from_db, config=config)
+        output_link_attr = html.escape(output_link, quote=True)
+
+        # Get version - try cmd_version field
+        version = html.escape(str(result.cmd_version if result.cmd_version else "-"), quote=True)
+        plugin_text = html.escape(str(result.plugin or ""), quote=True)
+        status_text = html.escape(str(status), quote=True)
+        pwd_text = html.escape(str(result.pwd or "-"), quote=True)
+
+        # Unique ID for this row's expandable output
+        row_id = f"output_{idx}_{str(result.id)[:8]}"
+        delete_button = ""
+        if can_delete:
+            delete_button = f'''
+                <button type="button" data-archive-result-ids="{result.id}" data-delete-url="{delete_url}"
+                        title="Delete this output">×</button>
+            '''
+
+        rows.append(f'''
+            <tr data-output-size="{output_size}" style="border-bottom: 1px solid #f1f5f9; transition: background 0.15s;" onmouseover="this.style.background='#f8fafc'" onmouseout="this.style.background='transparent'">
+                <td style="padding: 10px 12px; white-space: nowrap;">
+                    <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
+                       style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 11px;"
+                       title="View/edit archive result">
+                        <code>{str(result.id)[-8:]}</code>
+                    </a>
+                </td>
+                <td style="padding: 10px 12px; white-space: nowrap;">
+                    <span style="display: inline-block; padding: 3px 10px; border-radius: 12px;
+                                 font-size: 11px; font-weight: 600; text-transform: uppercase;
+                                 color: {color}; background: {bg};">{status_text}</span>
+                </td>
+                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{plugin_text}">
+                    {icon}
+                </td>
+                <td class="archive-results-plugin" style="padding: 10px 12px; font-weight: 500; color: #334155;">
+                        <a href="{output_link_attr}" target="_blank"
+                           style="color: #334155; text-decoration: none;"
+                       title="View output fullscreen"
+                       onmouseover="this.style.color='#2563eb'; this.style.textDecoration='underline';"
+                       onmouseout="this.style.color='#334155'; this.style.textDecoration='none';">
+                        {plugin_text}
+                    </a>
+                </td>
+                <td class="archive-results-output" style="padding: 10px 12px; max-width: 280px;">
+                    <span onclick="document.getElementById('{row_id}').open = !document.getElementById('{row_id}').open"
+                          style="display: block; max-width: 280px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
+                                 color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px; cursor: pointer;"
+                          title="Click to expand full output">
+                        {output_display}
+                    </span>
+                </td>
+                <td class="archive-results-files" style="padding: 10px 12px; color: #64748b; font-size: 12px; text-align: right;">
+                    {output_file_count}
+                </td>
+                <td class="archive-results-size" style="padding: 10px 12px; color: #64748b; font-size: 12px; text-align: right; white-space: nowrap;">
+                    {output_size_display}
+                </td>
+                <td class="archive-results-completed" style="padding: 10px 12px; color: #64748b; font-size: 12px;">
+                    {end_time}
+                </td>
+                <td style="padding: 10px 12px; white-space: nowrap;">
+                    {process_display}
+                </td>
+                <td style="padding: 10px 12px; white-space: nowrap;">
+                    {machine_display}
+                </td>
+                <td style="padding: 10px 12px; white-space: nowrap; font-family: ui-monospace, monospace; font-size: 11px; color: #64748b;">
+                    {version}
+                </td>
+                <td class="archive-results-actions-cell">
+                    <div class="archive-results-actions">
+                        <a href="{output_link_attr}" target="_blank"
+                           title="View output">📄</a>
+                        <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
+                           title="Edit">✏️</a>
+                        {delete_button}
+                    </div>
+                </td>
+            </tr>
+            <tr style="border-bottom: 1px solid #e2e8f0;">
+                <td colspan="12" style="padding: 0 12px 10px 12px;">
+                    <details id="{row_id}" style="margin: 0;">
+                        <summary style="cursor: pointer; font-size: 11px; color: #94a3b8; user-select: none;">
+                            Details &amp; Output
+                        </summary>
+                        <div style="margin-top: 8px; padding: 10px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; max-height: 200px; overflow: auto;">
+                            <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
+                                <span style="margin-right: 16px;"><b>ID:</b> <code>{result.id!s}</code></span>
+                                <span style="margin-right: 16px;"><b>Version:</b> <code>{version}</code></span>
+                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{pwd_text}</code></span>
+                            </div>
+                            <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
+                                <b>Output:</b>
+                            </div>
+                            <pre style="margin: 0; padding: 8px; background: #1e293b; border-radius: 4px; color: #e2e8f0; font-size: 12px; white-space: pre-wrap; word-break: break-all; max-height: 120px; overflow: auto;">{full_output}</pre>
+                            <div style="font-size: 11px; color: #64748b; margin-top: 8px;">
+                                <b>Command:</b>
+                            </div>
+                            <div style="position: relative; margin: 0; padding: 8px 56px 8px 8px; background: #1e293b; border-radius: 4px;">
+                                <button type="button"
+                                        data-command="{cmd_attr}"
+                                        onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
+                                        style="position: absolute; top: 6px; right: 6px; padding: 2px 8px; border: 0; border-radius: 4px; background: #334155; color: #e2e8f0; font-size: 11px; cursor: pointer;">
+                                    Copy
+                                </button>
+                                <code title="{cmd_attr}" style="display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: #e2e8f0; font-size: 11px;">{cmd_str_escaped}</code>
+                            </div>
+                        </div>
+                    </details>
+                </td>
+            </tr>
+        ''')
+
+    total_count = results[0]._inline_total_count
+    footer = ""
+    if total_count > limit:
+        footer = f"""
+            <tr data-output-footer>
+                <td colspan="12" style="padding: 12px; text-align: center; color: #64748b; font-size: 13px; background: #f8fafc;">
+                    Showing {limit} of {total_count} results &nbsp;
+                    <a href="/admin/core/archiveresult/?snapshot__id__exact={results[0].snapshot_id if results else ""}"
+                       style="color: #2563eb;">View all →</a>
+                </td>
+            </tr>
+        """
+
+    return mark_safe(f"""
+        <div style="border: 1px solid #e2e8f0; border-radius: 8px; overflow-x: auto; overflow-y: hidden; background: #fff; width: 100%;">
+            <table class="archive-results-table" style="width: 100%; min-width: 1100px; border-collapse: collapse; font-size: 14px;">
+                <thead>
+                    <tr style="background: #f8fafc; border-bottom: 2px solid #e2e8f0;">
+                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Details</th>
+                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Status</th>
+                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; width: 32px;"></th>
+                        <th class="archive-results-plugin" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Plugin</th>
+                        <th class="archive-results-output" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Output</th>
+                        <th class="archive-results-files" style="padding: 10px 12px; text-align: right; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Files</th>
+                        <th class="archive-results-size" style="padding: 10px 12px; text-align: right; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">
+                            <button type="button" data-output-size-sort style="all: unset; cursor: pointer;">Size ↕</button>
+                        </th>
+                        <th class="archive-results-completed" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Completed</th>
+                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Process</th>
+                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Machine</th>
+                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Version</th>
+                        <th class="archive-results-actions-cell" style="text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Actions</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {"".join(rows)}
+                    {footer}
+                </tbody>
+            </table>
+        </div>
+    """)
+
+
+class ArchiveResultInline(admin.TabularInline):
+    name = "Archive Results"
+    model = ArchiveResult
+    parent_model = Snapshot
+    extra = 0
+    sort_fields = ("end_ts", "plugin", "output_str", "status", "cmd_version")
+    readonly_fields = ("id", "result_id", "completed", "command", "version")
+    fields = ("start_ts", "end_ts", *readonly_fields, "plugin", "cmd", "cmd_version", "pwd", "status", "output_str")
+    ordering = ("end_ts",)
+    show_change_link = True
+
+    def get_parent_object_from_request(self, request):
+        resolved = resolve(request.path_info)
+        try:
+            return self.parent_model.objects.get(pk=resolved.kwargs["object_id"])
+        except (self.parent_model.DoesNotExist, ValidationError):
+            return None
+
+    @admin.display(
+        description="Completed",
+        ordering="end_ts",
+    )
+    def completed(self, obj):
+        return format_html('<p style="white-space: nowrap">{}</p>', obj.end_ts.strftime("%Y-%m-%d %H:%M:%S"))
+
+    def result_id(self, obj):
+        return format_html(
+            '<a href="{}"><code style="font-size: 10px">[{}]</code></a>',
+            reverse("admin:core_archiveresult_change", args=(obj.id,)),
+            str(obj.id)[:8],
         )
 
-    return mark_safe(
-        render_to_string(
-            "admin/core/archiveresult/inline.html",
-            {
-                "rows": rows,
-                "limit": limit,
-                "total_count": results[0]._inline_total_count,
-                "snapshot_id": results[0].snapshot_id,
-                "can_delete": can_delete,
-                "delete_url": reverse("admin:core_archiveresult_changelist"),
-            },
-        ),
-    )
+    def command(self, obj):
+        return format_html("<small><code>{}</code></small>", " ".join(obj.cmd or []))
+
+    def version(self, obj):
+        return format_html("<small><code>{}</code></small>", obj.cmd_version or "-")
+
+    def get_formset(self, request, obj=None, **kwargs):
+        formset = super().get_formset(request, obj, **kwargs)
+        snapshot = self.get_parent_object_from_request(request)
+        base_fields = formset.form.base_fields
+        snapshot_output_dir = str(snapshot.output_dir) if snapshot else ""
+
+        # formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
+
+        # default values for new entries
+        base_fields["status"].initial = "succeeded"
+        base_fields["start_ts"].initial = timezone.now()
+        base_fields["end_ts"].initial = timezone.now()
+        base_fields["cmd_version"].initial = "-"
+        base_fields["pwd"].initial = snapshot_output_dir
+        base_fields["cmd"].initial = '["-"]'
+        base_fields["output_str"].initial = "Manually recorded cmd output..."
+
+        if obj is not None:
+            # hidden values for existing entries and new entries
+            base_fields["start_ts"].widget = base_fields["start_ts"].hidden_widget()
+            base_fields["end_ts"].widget = base_fields["end_ts"].hidden_widget()
+            base_fields["cmd"].widget = base_fields["cmd"].hidden_widget()
+            base_fields["pwd"].widget = base_fields["pwd"].hidden_widget()
+            base_fields["cmd_version"].widget = base_fields["cmd_version"].hidden_widget()
+        return formset
+
+    def get_readonly_fields(self, request, obj=None):
+        if obj is not None:
+            return self.readonly_fields
+        else:
+            return []
 
 
 class ArchiveResultAdmin(BaseModelAdmin):
@@ -213,14 +464,40 @@ class ArchiveResultAdmin(BaseModelAdmin):
     autocomplete_fields = ("snapshot",)
 
     fieldsets = (
-        card_fieldset("Snapshot", ("snapshot", "snapshot_info", "tags_str", "admin_actions"), wide=True),
-        card_fieldset("Plugin", ("plugin_with_icon", "process_link", "status")),
-        card_fieldset("Timing", ("start_ts", "end_ts", "created_at", "modified_at")),
-        card_fieldset("Command", ("cmd", "cmd_str", "cmd_version", "pwd")),
-        card_fieldset(
+        (
+            "Snapshot",
+            {
+                "fields": ("snapshot", "snapshot_info", "tags_str", "admin_actions"),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Plugin",
+            {
+                "fields": ("plugin_with_icon", "process_link", "status"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timing",
+            {
+                "fields": ("start_ts", "end_ts", "created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Command",
+            {
+                "fields": ("cmd", "cmd_str", "cmd_version", "pwd"),
+                "classes": ("card",),
+            },
+        ),
+        (
             "Output",
-            ("output_str", "output_json", "output_files", "output_size", "output_mimetypes", "output_summary"),
-            wide=True,
+            {
+                "fields": ("output_str", "output_json", "output_files", "output_size", "output_mimetypes", "output_summary"),
+                "classes": ("card", "wide"),
+            },
         ),
     )
 
@@ -283,8 +560,8 @@ class ArchiveResultAdmin(BaseModelAdmin):
             try:
                 queryset = self.get_queryset(request).filter(pk__in=selected)
                 if not queryset.exists():
-                    snapshot = Snapshot.objects.only("id").filter(pk=request.GET.get("snapshot")).first()
-                    return redirect(build_snapshot_url(str(snapshot.id), "index.html", request=request) if snapshot else request.path)
+                    snapshot = Snapshot.objects.filter(pk=request.GET.get("snapshot")).first()
+                    return redirect(build_snapshot_detail_url(snapshot.archive_path_from_db, request=request) if snapshot else request.path)
             except (ValidationError, ValueError):
                 return redirect(request.path)
             return delete_selected(self, request, queryset)
@@ -296,7 +573,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
             request.GET.clear()
             request.META["QUERY_STRING"] = ""
             try:
-                handoff_snapshot = Snapshot.objects.only("id").filter(pk=handoff_snapshot).first()
+                handoff_snapshot = Snapshot.objects.filter(pk=handoff_snapshot).first()
             except (ValidationError, ValueError):
                 handoff_snapshot = None
         saved_list_per_page = self.list_per_page
@@ -310,13 +587,24 @@ class ArchiveResultAdmin(BaseModelAdmin):
                     pk__in=request.POST.getlist(ACTION_CHECKBOX_NAME),
                 ).exists()
             ):
-                return redirect(build_snapshot_url(str(handoff_snapshot.id), "index.html", request=request))
+                return redirect(build_snapshot_detail_url(handoff_snapshot.archive_path_from_db, request=request))
             return response
         finally:
             self.list_per_page = saved_list_per_page
 
     def get_queryset(self, request):
-        ordering_fields = self.get_ordering_fields(request)
+        ordering = request.GET.get("o")
+        ordering_fields = set()
+        if ordering:
+            for part in ordering.split("."):
+                if not part:
+                    continue
+                try:
+                    idx = abs(int(part)) - 1
+                except ValueError:
+                    continue
+                if 0 <= idx < len(self.list_display):
+                    ordering_fields.add(self.list_display[idx])
 
         qs = (
             super()
@@ -373,7 +661,11 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     def get_snapshot_view_url(self, result: ArchiveResult) -> str:
         request = self.request
-        return build_snapshot_url(str(result.snapshot_id), request=request, config=request.archivebox_config)
+        return build_snapshot_detail_url(
+            result.snapshot.archive_path_from_db,
+            request=request,
+            config=request.archivebox_config,
+        )
 
     def get_output_view_url(self, result: ArchiveResult) -> str:
         request = self.request
@@ -385,10 +677,21 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     def get_output_files_url(self, result: ArchiveResult) -> str:
         request = self.request
-        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=request.archivebox_config)}/?files=1"
+        return build_snapshot_files_url(
+            str(result.snapshot_id),
+            result.plugin,
+            request=request,
+            config=request.archivebox_config,
+        )
 
     def get_output_zip_url(self, result: ArchiveResult) -> str:
-        return f"{self.get_output_files_url(result)}&download=zip"
+        request = self.request
+        return build_snapshot_zip_url(
+            str(result.snapshot_id),
+            result.plugin,
+            request=request,
+            config=request.archivebox_config,
+        )
 
     @admin.display(description="Details", ordering="id")
     def details_link(self, result):
@@ -446,7 +749,11 @@ class ArchiveResultAdmin(BaseModelAdmin):
         request = self.request
         return format_html(
             '<a href="{}"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
+            build_snapshot_detail_url(
+                result.snapshot.archive_path_from_db,
+                request=request,
+                config=request.archivebox_config,
+            ),
             snapshot_id[:8],
             result.snapshot.bookmarked_at.strftime("%Y-%m-%d %H:%M"),
             result.snapshot.url[:128],
@@ -517,9 +824,26 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     @admin.display(description="Command")
     def cmd_str(self, result):
-        return render_copy_block(
-            build_abx_dl_display_command(result),
-            copy_text=build_abx_dl_replay_command(result, config=self.request.archivebox_config),
+        request = self.request
+        display_cmd = build_abx_dl_display_command(result)
+        replay_cmd = build_abx_dl_replay_command(result, config=request.archivebox_config)
+        return format_html(
+            """
+            <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
+                <button type="button"
+                        data-command="{}"
+                        onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
+                        style="position: absolute; top: 6px; right: 6px; z-index: 1; padding: 2px 8px; border: 0; border-radius: 4px; background: #e2e8f0; color: #334155; font-size: 11px; cursor: pointer;">
+                    Copy
+                </button>
+                <code title="{}" style="display: block; width: 100%; max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 8px 56px 8px 8px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; font-size: 11px; box-sizing: border-box;">
+                    {}
+                </code>
+            </div>
+            """,
+            replay_cmd,
+            replay_cmd,
+            display_cmd,
         )
 
     def output_display(self, result):
@@ -527,11 +851,14 @@ class ArchiveResultAdmin(BaseModelAdmin):
         config = request.archivebox_config
         # Determine output link path - use embed_path() which checks output_files
         embed_path = result.embed_path()
-        output_path = embed_path if (result.status == "succeeded" and embed_path) else "index.html"
-        snapshot_id = str(result.snapshot_id)
+        output_url = (
+            build_snapshot_url(str(result.snapshot_id), embed_path, request=request, config=config)
+            if result.status == "succeeded" and embed_path
+            else build_snapshot_detail_url(result.snapshot.archive_path_from_db, request=request, config=config)
+        )
         return format_html(
             '<a href="{}" class="output-link">↗️</a><pre>{}</pre>',
-            build_snapshot_url(snapshot_id, output_path, request=request, config=config),
+            output_url,
             result.output_str_for_display(),
         )
 
@@ -563,11 +890,15 @@ class ArchiveResultAdmin(BaseModelAdmin):
             '<pre style="display: inline-block">{}</pre><br/>',
             result.output_str_for_display(),
         )
-        snapshot_id = str(result.snapshot_id)
         request = self.request
         output_html += format_html(
-            '<a href="{}#all">See result files ...</a><br/><pre><code>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
+            '<a href="{}">See result files ...</a><br/><pre><code>',
+            build_snapshot_detail_url(
+                result.snapshot.archive_path_from_db,
+                output_path="all",
+                request=request,
+                config=request.archivebox_config,
+            ),
         )
         embed_path = result.embed_path() or ""
         path_from_embed = snapshot_dir / (embed_path or "")

archivebox/core/admin_snapshots.py

@@ -1,46 +1,62 @@
 __package__ = "archivebox.core"
 
 import json
-from copy import copy
-from urllib.parse import urlencode
-from functools import lru_cache
-from types import SimpleNamespace
+from math import ceil
+from typing import Any
 
 from django.contrib import admin, messages
+from django.core.exceptions import PermissionDenied
 from django.urls import path, reverse
 from django.shortcuts import get_object_or_404, redirect
-from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed
+from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed, Http404
 from django.utils import timezone
-from django.utils.html import format_html
+from django.utils.formats import date_format
+from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
 from django.db.models import Q, Count, Exists, F, OuterRef, Prefetch
 from django import forms
-from django.template import Template, RequestContext
-from django.template.loader import render_to_string
+from django.template import Context, Template, RequestContext
 from django.contrib.admin.helpers import ActionForm
 
 from archivebox.config.common import get_config
 from archivebox.misc.util import htmldecode, urldecode
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.misc.logging_util import printable_filesize
+from archivebox.misc.serve_static import serve_static_with_byterange_support
 from archivebox.search.admin import SearchResultsAdminMixin, SearchResultsChangeList
 from archivebox.search.views import admin_snapshot_search_stream_view
-from archivebox.core.routes_util import build_snapshot_url, build_web_url
+from archivebox.core.routes_util import (
+    build_snapshot_detail_url,
+    build_snapshot_files_url,
+    build_snapshot_role_output_url,
+    build_snapshot_url,
+    build_snapshot_zip_url,
+    get_snapshot_output_anchor,
+)
+from archivebox.core.tag_util import get_or_create_tag
+from archivebox.core.templatetags.core_tags import snapshot_thumbnail, snapshot_url_text
 from archivebox.plugins.hooks import discover_hooks
-from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_plugins
+from archivebox.plugins.discovery import get_plugin_icon, get_snapshot_role_names
+from archivebox.plugins.output_groups import (
+    OUTPUT_GROUPS,
+    display_plugin_name,
+    output_group_for_plugin,
+    order_output_plugins,
+    plugin_icon_is_hidden,
+    plugin_output_sizes,
+)
 
-from archivebox.core.widgets import render_permissions_badge, render_snapshot_progress
-from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
+from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 
 from archivebox.core.models import Tag, Snapshot, ArchiveResult
 from archivebox.crawls.models import Crawl
 from archivebox.core.admin_archiveresults import render_archiveresults_list
-from archivebox.core.preview_util import EXTENSION_SCREENSHOT_PLUGIN
 from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
     PERMISSIONS_META,
     get_snapshot_permissions,
+    normalize_permissions,
 )
 from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget
 
@@ -50,9 +66,12 @@ GLOBAL_CONTEXT = {}
 SNAPSHOT_PERMISSION_META = PERMISSIONS_META
 
 
-@lru_cache(maxsize=1)
-def _plugin_sort_order() -> dict[str, int]:
-    return {get_plugin_name(plugin): idx for idx, plugin in enumerate(get_plugins())}
+def _format_size_column(num_bytes: int) -> str:
+    size = num_bytes
+    for unit in ("B", "KB", "MB", "GB", "TB"):
+        if size < 1024 or unit == "TB":
+            return f"{size:.1f} {unit}" if unit == "GB" else f"{ceil(size)} {unit}"
+        size /= 1024
 
 
 class SnapshotActionForm(ActionForm):
@@ -232,12 +251,10 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
 
 
 class SnapshotChangeList(SearchResultsChangeList):
-    snapshot_status_choices = Snapshot.StatusChoices.choices
-
     def __init__(self, request, *args, **kwargs):
         super().__init__(request, *args, **kwargs)
         resolver_name = request.resolver_match.url_name
-        self.embedded_changelist = request.GET.get("_embedded") in {"crawl", "snapshot"}
+        self.embedded_changelist = request.GET.get("_embedded") == "crawl"
         self.snapshot_is_grid_view = not self.embedded_changelist and (
             resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")
         )
@@ -266,15 +283,28 @@ class SnapshotChangeList(SearchResultsChangeList):
         rows = (
             ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
             .order_by("snapshot_id", "plugin")
-            .values_list("snapshot_id", "plugin", "status", "output_size", "output_files")
+            .only("snapshot_id", "plugin", "status", "output_str", "output_size", "output_files")
         )
-        for snapshot_id, plugin, status, output_size, output_files in rows.iterator(chunk_size=1000):
-            if plugin in seen_plugins[snapshot_id]:
+        for result in rows.iterator(chunk_size=1000):
+            if result.plugin in seen_plugins[result.snapshot_id]:
                 continue
-            seen_plugins[snapshot_id].add(plugin)
-            output_results_by_snapshot[snapshot_id].append(
-                SimpleNamespace(plugin=plugin, status=status, output_size=output_size, output_files=output_files),
+            seen_plugins[result.snapshot_id].add(result.plugin)
+            output_results_by_snapshot[result.snapshot_id].append(result)
+
+        card_results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
+        for result in (
+            ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids)
+            .exclude(plugin="")
+            .only(
+                "snapshot_id",
+                "plugin",
+                "status",
+                "output_str",
+                "output_size",
+                "output_files",
             )
+        ):
+            card_results_by_snapshot[result.snapshot_id].append(result)
 
         for obj in self.result_list:
             counts = status_counts_by_snapshot.get(obj.pk, {})
@@ -298,10 +328,11 @@ class SnapshotChangeList(SearchResultsChangeList):
                 "is_sealed": obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED, obj.StatusChoices.PAUSED),
             }
             obj.__dict__["_admin_output_results"] = output_results_by_snapshot[obj.pk]
+            obj.__dict__["_snapshot_card_results"] = card_results_by_snapshot[obj.pk]
 
     def get_results(self, request):
         super().get_results(request)
-        if request.GET.get("_embedded") in {"crawl", "snapshot"}:
+        if request.GET.get("_embedded") == "crawl":
             self.full_result_count = self.result_count
             self.show_full_result_count = True
         self._attach_archiveresult_summaries()
@@ -310,6 +341,7 @@ class SnapshotChangeList(SearchResultsChangeList):
 class SnapshotAdminForm(forms.ModelForm):
     """Custom form for Snapshot admin with tag editor widget."""
 
+    url = forms.CharField(widget=forms.TextInput)
     tags_editor = forms.CharField(
         label="Tags",
         required=False,
@@ -372,7 +404,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     list_display = (
         "permissions_badge",
-        "created_at",
+        "created_at_display",
         "preview_icon",
         "title_str",
         "tags_inline",
@@ -380,7 +412,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         "files",
         "size_with_stats",
     )
-    list_display_links = ("created_at",)
+    list_display_links = ("created_at_display",)
     sort_fields = ("title_str", "created_at", "status", "crawl")
     readonly_fields = (
         "admin_actions",
@@ -415,19 +447,70 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                 "classes": ("card", "actions-card"),
             },
         ),
-        card_fieldset("Snapshot", ("snapshot_summary",)),
-        card_fieldset("URL", (("url_favicon", "url"), ("title", "tags_badges")), wide=True),
-        card_fieldset("Tags", ("tags_editor", "permissions_config")),
-        card_fieldset("Status", ("status", "retry_at")),
-        card_fieldset("Timestamps", ("bookmarked_at", "created_at", "modified_at", "downloaded_at")),
-        card_fieldset("Relations", ("crawl",)),
-        card_fieldset(
+        (
+            "Snapshot",
+            {
+                "fields": ("snapshot_summary",),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "URL",
+            {
+                "fields": (("url_favicon", "url"), ("title", "tags_badges")),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Tags",
+            {
+                "fields": ("tags_editor", "permissions_config"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Status",
+            {
+                "fields": ("status", "retry_at"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": ("bookmarked_at", "created_at", "modified_at", "downloaded_at"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Relations",
+            {
+                "fields": ("crawl",),
+                "classes": ("card",),
+            },
+        ),
+        (
             "Config",
-            ("config",),
-            description='<span style="display:block; margin:-4px 0 6px; font-size:11px; line-height:1.35; color:#94a3b8;">Uses <code>Crawl.config</code> by default. Only set per-snapshot overrides here when needed.</span>',
+            {
+                "fields": ("config",),
+                "description": '<span style="display:block; margin:-4px 0 6px; font-size:11px; line-height:1.35; color:#94a3b8;">Uses <code>Crawl.config</code> by default. Only set per-snapshot overrides here when needed.</span>',
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Files",
+            {
+                "fields": ("output_dir",),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Archive Results",
+            {
+                "fields": ("archiveresults_list",),
+                "classes": ("card", "wide"),
+            },
         ),
-        card_fieldset("Files", ("output_dir",)),
-        card_fieldset("Archive Results", ("archiveresults_list",), wide=True),
     )
 
     ordering = ["-created_at"]
@@ -443,6 +526,15 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     inlines = []  # Removed TagInline, using TagEditorWidget instead
     list_per_page = 50
 
+    @admin.display(description="Date", ordering="created_at")
+    def created_at_display(self, obj):
+        created_at = timezone.localtime(obj.created_at)
+        return format_html(
+            '<span class="snapshot-created-at"><span class="snapshot-created-at__date">{}</span><span class="snapshot-created-at__time">{}</span></span>',
+            date_format(created_at, "Y-m-d"),
+            date_format(created_at, "g:i A"),
+        )
+
     action_form = SnapshotActionForm
     paginator = AcceleratedPaginator
 
@@ -473,7 +565,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def changelist_view(self, request, extra_context=None):
         self.request = request
         saved_list_per_page = self.list_per_page
-        embedded_changelist = request.GET.get("_embedded") in {"crawl", "snapshot"}
+        embedded_changelist = request.GET.get("_embedded") == "crawl"
         if embedded_changelist:
             try:
                 requested_per_page = int(request.GET.get("per_page", "200"))
@@ -494,27 +586,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         finally:
             self.list_per_page = saved_list_per_page
 
-    def render_embedded_changelist(self, request, *, filters, title, default_search_mode=None):
-        """Reuse snapshot search, rows, pagination, and actions in parent admin pages."""
-        changelist_path = reverse(f"{self.admin_site.name}:core_snapshot_changelist")
-        full_url = f"{changelist_path}?{urlencode(filters)}"
-        changelist_request = copy(request)
-        changelist_request.method = "GET"
-        changelist_request.path = changelist_path
-        changelist_request.GET = request.GET.copy()
-        if default_search_mode:
-            changelist_request.GET.setdefault("search_mode", default_search_mode)
-        changelist_request.GET.update({**filters, "_embedded": "snapshot", "per_page": "200"})
-        changelist_request.POST = request.POST.copy()
-        changelist_request.POST.clear()
-        response = self.changelist_view(changelist_request)
-        context = {
-            **response.context_data,
-            "snapshot_changelist_url": full_url,
-            "snapshot_changelist_title": title,
-        }
-        return mark_safe(render_to_string("admin/core/snapshot/embedded_changelist.html", context, request=request))
-
     def get_actions(self, request):
         actions = super().get_actions(request)
         if not actions:
@@ -523,26 +594,32 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return actions
 
     def lookup_allowed(self, lookup, value, request=None):
-        if lookup in {"crawl__id__exact", "crawl_id__exact", "crawl_id", "crawl__schedule__id__exact"}:
+        if lookup in {"crawl__id__exact", "crawl_id__exact", "crawl_id"}:
             return True
         return super().lookup_allowed(lookup, value, request=request)
 
     def get_snapshot_view_url(self, obj: Snapshot) -> str:
         request = self.request
-        return build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)
+        return build_snapshot_detail_url(obj.archive_path_from_db, request=request, config=request.archivebox_config)
 
     def get_snapshot_files_url(self, obj: Snapshot) -> str:
         request = self.request
-        return f"{build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)}/?files=1"
+        return build_snapshot_files_url(str(obj.id), request=request, config=request.archivebox_config)
 
     def get_snapshot_zip_url(self, obj: Snapshot) -> str:
-        return f"{self.get_snapshot_files_url(obj)}&download=zip"
+        request = self.request
+        return build_snapshot_zip_url(str(obj.id), request=request, config=request.archivebox_config)
 
     def get_urls(self):
         urls = super().get_urls()
         custom_urls = [
             path("grid/", self.admin_site.admin_view(self.grid_view), name="grid"),
             path("search-stream/", self.admin_site.admin_view(self.search_stream_view), name="core_snapshot_search_stream"),
+            path(
+                "<path:object_id>/preview/<str:plugin>/<path:filename>",
+                self.admin_site.admin_view(self.preview_view, cacheable=True),
+                name="core_snapshot_preview",
+            ),
             path("<path:object_id>/redo-failed/", self.admin_site.admin_view(self.redo_failed_view), name="core_snapshot_redo_failed"),
             path(
                 "<path:object_id>/set-permissions/",
@@ -552,6 +629,26 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         ]
         return custom_urls + urls
 
+    def preview_view(self, request, object_id, plugin, filename):
+        """Serve a saved image output from the authenticated admin origin."""
+        snapshot = get_object_or_404(Snapshot, pk=object_id)
+        if not self.has_view_or_change_permission(request, snapshot):
+            raise PermissionDenied
+        result = snapshot.archiveresult_set.filter(plugin=plugin, status=ArchiveResult.StatusChoices.SUCCEEDED).first()
+        try:
+            output_path = result.output_file_path(filename) if result else None
+        except (TypeError, ValueError, OverflowError):
+            output_path = None
+        if not output_path:
+            raise Http404("Snapshot image does not exist")
+        request.archivebox_cache_policy = "private"
+        response = serve_static_with_byterange_support(request, output_path, document_root=snapshot.output_dir)
+        if response.status_code != 304 and not response.headers.get("Content-Type", "").lower().startswith("image/"):
+            raise Http404("Snapshot preview is not an image")
+        response.headers["Vary"] = "Cookie"
+        response.headers["X-Content-Type-Options"] = "nosniff"
+        return response
+
     def search_stream_view(self, request):
         return admin_snapshot_search_stream_view(self, request)
 
@@ -624,7 +721,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     def get_queryset(self, request):
         self.request = request
-        ordering_fields = self.get_ordering_fields(request)
+        ordering_fields = self._get_ordering_fields(request)
         needs_files_sort = "files" in ordering_fields
         needs_tags_sort = "tags_inline" in ordering_fields
         is_change_view = request.resolver_match.url_name == "core_snapshot_change"
@@ -683,15 +780,50 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
         return qs
 
-    @admin.display(description="👁", ordering="permissions")
+    @admin.display(description="🔒", ordering="permissions")
     def permissions_badge(self, obj):
         permissions = obj.__dict__.get("snapshot_permissions")
         if permissions is None:
             permissions = obj.permissions
-        return render_permissions_badge(
+        permissions = normalize_permissions(permissions)
+        icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
+        menu_items = format_html_join(
+            "",
+            (
+                '<button type="button" class="snapshot-permissions-menu-item{}" data-permissions="{}">'
+                '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
+                "<span>{}</span>"
+                "</button>"
+            ),
+            (
+                (
+                    " is-active" if choice_value == permissions else "",
+                    choice_value,
+                    choice_fg,
+                    choice_bg,
+                    choice_icon,
+                    choice_label,
+                )
+                for choice_value, choice_label in PERMISSIONS_CHOICES
+                for choice_icon, _choice_title, choice_fg, choice_bg in [SNAPSHOT_PERMISSION_META[choice_value]]
+            ),
+        )
+        return format_html(
+            '<span class="snapshot-permissions-quick" data-current-permissions="{}" data-permissions-url="{}">'
+            '<button type="button" class="snapshot-permissions-button snapshot-permissions-{}" title="{}" aria-label="Change snapshot permissions: {}" aria-expanded="false">'
+            '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
+            "</button>"
+            '<span class="snapshot-permissions-menu" role="menu" hidden>{}</span>'
+            "</span>",
             permissions,
-            url=reverse(f"{self.admin_site.name}:core_snapshot_set_permissions", args=[obj.pk]),
-            object_name="snapshot",
+            reverse(f"{self.admin_site.name}:core_snapshot_set_permissions", args=[obj.pk]),
+            permissions,
+            label,
+            label,
+            fg,
+            bg,
+            icon,
+            menu_items,
         )
 
     @admin.display(description="Imported Timestamp")
@@ -791,6 +923,29 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             obj.pk,
         )
 
+    def status_info(self, obj):
+        request = self.request
+        config = request.archivebox_config
+        favicon_url = build_snapshot_role_output_url(
+            obj,
+            "list_icon",
+            request=request,
+            config=config,
+            fallback_to_default=True,
+        )
+        return format_html(
+            """
+            Archived: {} ({} files {}) &nbsp; &nbsp;
+            Favicon: <img src="{}" style="height: 20px"/> &nbsp; &nbsp;
+            Extension: {} &nbsp; &nbsp;
+            """,
+            "✅" if obj.is_archived else "❌",
+            obj.num_outputs,
+            self.size(obj) or "0kb",
+            favicon_url,
+            obj.extension or "-",
+        )
+
     @admin.display(description="Archive Results")
     def archiveresults_list(self, obj):
         request = self.request
@@ -815,7 +970,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         show_title = bool(title_raw) and title_normalized != "pending..." and title_normalized != url_normalized
         css_class = "fetched" if show_title else "pending"
 
-        detail_url = build_web_url(f"/{obj.archive_path_from_db}/index.html", request=request, config=config)
+        detail_url = build_snapshot_detail_url(obj.archive_path_from_db, request=request, config=config)
         title_html = ""
         if show_title:
             title_html = format_html(
@@ -825,14 +980,26 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                 urldecode(htmldecode(title_raw))[:128],
             )
 
+        favicon = self._get_favicon_data(obj) or {}
+        favicon_html = (
+            format_html(
+                '<img class="snapshot-url-favicon" src="{}" alt="" loading="lazy" decoding="async" onerror="this.hidden=true">',
+                favicon["url"],
+            )
+            if favicon.get("url")
+            else ""
+        )
         return format_html(
+            '<a class="snapshot-title-detail-hitbox" href="{}" aria-label="Open snapshot details"></a>'
             "{}"
-            '<div style="font-size: 11px; color: #64748b; margin-top: 2px;">'
-            '<a href="{}"><code style="user-select: all;">{}</code></a>'
+            '<div class="snapshot-title-url">'
+            '<a class="snapshot-original-url" href="{}">{}<code style="user-select: all;">{}</code></a>'
             "</div>",
+            detail_url,
             title_html,
             url_raw or obj.url,
-            (url_raw or obj.url)[:128],
+            favicon_html,
+            snapshot_url_text(url_raw or obj.url),
         )
 
     @admin.display(description="Tags", ordering="tag_count")
@@ -859,90 +1026,47 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
         return mark_safe(f'<span class="tags-inline-editor">{tags_html}</span>')
 
-    def _get_preview_data(self, obj):
-        request = self.request
-        config = request.archivebox_config
+    def _get_favicon_data(self, obj):
+        list_icon_plugins = set(get_snapshot_role_names("list_icon"))
         results = self._get_prefetched_results(obj)
-        if results is not None:
-            results = [r for r in results if r.plugin in ("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")]
-        else:
-            results = list(
-                obj.archiveresult_set.filter(plugin__in=("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")).only(
-                    "snapshot_id",
-                    "plugin",
-                    "status",
-                    "output_files",
-                    "output_str",
-                ),
+        if results is None:
+            results = obj.archiveresult_set.filter(
+                plugin__in=list_icon_plugins,
+                status=ArchiveResult.StatusChoices.SUCCEEDED,
             )
+        urls = []
+        for result in results:
+            if result.plugin not in list_icon_plugins or result.status != ArchiveResult.StatusChoices.SUCCEEDED:
+                continue
+            try:
+                output_path = result.embed_path()
+            except (TypeError, ValueError, OverflowError):
+                continue
+            if output_path:
+                urls.append(
+                    build_snapshot_url(str(obj.id), output_path, request=self.request, config=self.request.archivebox_config),
+                )
+        return {"url": urls[0], "fallbacks": ",".join(urls[1:])} if urls else None
 
-        def result_output_path(result, filename: str) -> str | None:
-            output_files = result.output_files or {}
-            file_info = output_files.get(filename)
-            if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
-                return None
-            if file_info.get("root_relative"):
-                return filename
-            return f"{result.plugin}/{filename}"
-
-        def result_urls(plugin: str, filenames: tuple[str, ...]) -> list[str]:
-            urls: list[str] = []
-            for result in results:
-                if result.plugin != plugin or result.status != ArchiveResult.StatusChoices.SUCCEEDED:
-                    continue
-                for filename in filenames:
-                    output_path = result_output_path(result, filename)
-                    if output_path:
-                        urls.append(build_snapshot_url(str(obj.id), output_path, request=request, config=config))
-            return urls
-
-        screenshot_urls = result_urls("screenshot", ("screenshot.png",))
-        extension_screenshot_urls = result_urls(EXTENSION_SCREENSHOT_PLUGIN, ("screenshot-1.png", "screenshot.png"))
-        favicon_urls = result_urls("favicon", ("favicon.ico",))
-
-        if not screenshot_urls and not extension_screenshot_urls and not favicon_urls:
-            return None
-
-        all_screenshot_urls = [*screenshot_urls, *extension_screenshot_urls]
-        if all_screenshot_urls:
-            img_url = all_screenshot_urls[0]
-            fallbacks = [*all_screenshot_urls[1:], *favicon_urls]
-            img_alt = "Screenshot"
-            preview_class = "screenshot"
-        else:
-            img_url = favicon_urls[0]
-            fallbacks = favicon_urls[1:]
-            img_alt = "Favicon"
-            preview_class = "favicon"
-
-        fallback_list = ",".join(fallbacks)
-        onerror_js = (
-            "this.dataset.fallbacks && this.dataset.fallbacks.length ? "
-            "(this.src=this.dataset.fallbacks.split(',').shift(), "
-            "this.dataset.fallbacks=this.dataset.fallbacks.split(',').slice(1).join(',')) : "
-            "this.remove()"
+    def _render_thumbnail(self, obj, width="100px", height="100px"):
+        return snapshot_thumbnail(
+            Context({"request": self.request, "CONFIG": self.request.archivebox_config}),
+            obj,
+            admin=True,
+            width=width,
+            height=height,
         )
 
-        return {
-            "img_url": img_url,
-            "img_alt": img_alt,
-            "preview_class": preview_class,
-            "onerror_js": onerror_js,
-            "fallback_list": fallback_list,
-            "favicon_url": favicon_urls[0] if favicon_urls else "",
-            "favicon_fallback_list": ",".join(favicon_urls[1:]),
-        }
-
     @admin.display(description="", empty_value="")
     def url_favicon(self, obj):
-        preview = self._get_preview_data(obj)
-        if not preview:
+        favicon = self._get_favicon_data(obj)
+        if not favicon:
             return ""
 
-        favicon_url = preview.get("favicon_url") or ""
+        favicon_url = favicon.get("url") or ""
         if not favicon_url:
             return ""
-        fallback_list = preview.get("favicon_fallback_list") or ""
+        fallback_list = favicon.get("fallbacks") or ""
         onerror_js = (
             "this.dataset.fallbacks && this.dataset.fallbacks.length ? "
             "(this.src=this.dataset.fallbacks.split(',').shift(), "
@@ -963,42 +1087,24 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Preview", empty_value="")
     def preview_icon(self, obj):
-        preview = self._get_preview_data(obj)
-        if not preview:
-            return None
-
         return format_html(
-            '<img src="{}" alt="{}" class="snapshot-preview {}" decoding="async" loading="lazy" onerror="{}" data-fallbacks="{}">',
-            preview["img_url"],
-            preview["img_alt"],
-            preview["preview_class"],
-            preview["onerror_js"],
-            preview["fallback_list"],
+            '<a href="{}" title="Open snapshot details">{}</a>',
+            build_snapshot_detail_url(obj.archive_path_from_db, request=self.request, config=self.request.archivebox_config),
+            self._render_thumbnail(obj, height="38px"),
         )
 
     @admin.display(description=" ", empty_value="")
     def snapshot_summary(self, obj):
         request = self.request
         config = request.archivebox_config
-        preview = self._get_preview_data(obj)
         stats = self._get_progress_stats(obj)
         archive_size = stats["output_size"] or 0
         size_txt = printable_filesize(archive_size) if archive_size else "pending"
-        screenshot_html = ""
-
-        if preview:
-            screenshot_html = format_html(
-                '<a href="{href}" title="Open snapshot live view" style="display:block; flex:0 0 220px; width:220px;">'
-                '<img src="{src}" alt="{alt}" decoding="async" loading="lazy" onerror="{onerror}" data-fallbacks="{fallbacks}" '
-                'style="display:block; width:100%; max-width:220px; aspect-ratio: 16 / 10; object-fit: cover; object-position: top; '
-                'border-radius: 10px; border: 1px solid #e2e8f0; background: #f8fafc;">'
-                "</a>",
-                href=build_web_url(f"/{obj.archive_path}", request=request, config=config),
-                src=preview["img_url"],
-                alt=preview["img_alt"],
-                onerror=preview["onerror_js"],
-                fallbacks=preview["fallback_list"],
-            )
+        screenshot_html = format_html(
+            '<a href="{}" title="Open snapshot live view" style="display:block;flex:0 0 220px;">{}</a>',
+            build_snapshot_detail_url(obj.archive_path_from_db, request=request, config=config),
+            self._render_thumbnail(obj, width="220px", height="138px"),
+        )
 
         return format_html(
             '<div style="display:flex; gap:16px; align-items:flex-start;">'
@@ -1013,18 +1119,19 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             "</div>",
             screenshot_html,
             size_txt,
-            build_web_url(f"/{obj.archive_path}", request=request, config=config),
-            obj.archive_path,
+            self.get_snapshot_files_url(obj),
+            obj.archive_path_from_db,
         )
 
     @admin.display(
         description="Files Saved",
         ordering="ar_succeeded_count",
+        empty_value="",
     )
     def files(self, obj):
         results = self._get_prefetched_results(obj)
         if results is None:
-            results = obj.archiveresult_set.only("plugin", "status", "output_size")
+            results = obj.archiveresult_set.only("plugin", "status", "output_size", "output_files")
 
         plugins_with_output: dict[str, ArchiveResult] = {}
         for result in results:
@@ -1035,56 +1142,111 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             plugins_with_output.setdefault(result.plugin, result)
 
         if not plugins_with_output:
-            return mark_safe('<span style="opacity: 0.35;">...</span>')
+            return ""
 
-        sorted_results = sorted(
-            plugins_with_output.values(),
-            key=lambda result: (_plugin_sort_order().get(result.plugin, 9999), result.plugin),
-        )
-        visible_results = sorted_results[:14]
-        output = []
+        sorted_results = [plugins_with_output[plugin] for plugin in order_output_plugins(plugins_with_output, plugin_output_sizes(results))]
+        icons_by_group: dict[str, list[dict[str, Any]]] = {group_id: [] for group_id, _, _ in OUTPUT_GROUPS}
         request = self.request
         config = request.archivebox_config
-        for result in visible_results:
+        for result in sorted_results:
+            if plugin_icon_is_hidden(result.plugin):
+                continue
             icon = mark_safe(get_plugin_icon(result.plugin))
             if not icon.strip():
                 continue
-            output.append(
-                format_html(
-                    '<a href="{}" class="exists-True" title="{}">{}</a>',
-                    build_web_url(f"/{obj.archive_path_from_db}/{result.plugin}/", request=request, config=config),
-                    result.plugin,
-                    icon,
-                ),
-            )
-        if len(sorted_results) > len(visible_results):
-            output.append(
-                format_html(
-                    '<span title="{} more outputs">+{}</span>',
-                    len(sorted_results) - len(visible_results),
-                    len(sorted_results) - len(visible_results),
-                ),
+            embed_path = result.embed_path()
+            if not embed_path:
+                continue
+            group_id = output_group_for_plugin(result.plugin)
+            anchor_path = get_snapshot_output_anchor(result.plugin, embed_path)
+            icons_by_group[group_id].append(
+                {
+                    "plugin": result.plugin,
+                    "icon": icon,
+                    "href": build_snapshot_detail_url(
+                        obj.archive_path_from_db,
+                        output_path=anchor_path,
+                        request=request,
+                        config=config,
+                    ),
+                },
             )
 
+        output = []
+        for group_id, label, _ in OUTPUT_GROUPS:
+            icons = icons_by_group[group_id]
+            if icons:
+                top = icons[0]
+                front_index = 0
+                pile_column = len(output)
+                top_html = format_html(
+                    '<a href="{}" class="exists-True files-icon-pile-top files-icon-pile-cover files-icon-plugin--{}" aria-label="{}" data-tooltip="{}" style="--files-icon-front-index:{};--files-icon-front-offset:{}px">{}</a>',
+                    top["href"],
+                    top["plugin"],
+                    label,
+                    label,
+                    front_index,
+                    front_index * 2,
+                    top["icon"],
+                )
+                members = []
+                for icon_index, item in enumerate(icons):
+                    row = icon_index // 5
+                    row_count = min(5, len(icons) - row * 5)
+                    member_top = 8 + row * 18
+                    row_offset = (5 - row_count) * 11 + 3
+                    member_left = 3 + row_offset + (icon_index % 5) * 22
+                    members.append(
+                        format_html(
+                            '<a href="{}" class="exists-True files-icon-plugin--{}" aria-label="{}" data-tooltip="{}" style="--files-icon-index:{};--files-icon-row:{};--files-icon-column:{};--files-icon-row-offset:{}px;--files-icon-member-left:{}px;--files-icon-member-top:{}px">{}</a>',
+                            item["href"],
+                            item["plugin"],
+                            display_plugin_name(item["plugin"]),
+                            item["plugin"],
+                            icon_index,
+                            row,
+                            icon_index % 5,
+                            row_offset,
+                            member_left,
+                            member_top,
+                            item["icon"],
+                        ),
+                    )
+                popup = format_html(
+                    '<span class="files-icon-pile-popup" popover="manual">{}<span class="files-icon-pile-label">{}</span></span>',
+                    mark_safe("".join(str(icon) for icon in members)),
+                    label,
+                )
+                pile_contents = format_html("{}{}", top_html, popup)
+                output.append(
+                    format_html(
+                        '<span class="files-icon-pile files-icon-pile--{}" role="group" aria-label="{} saved outputs" style="--files-icon-popup-left:{}px;--files-icon-rows:{};--files-icon-card-width:{}px">{}</span>',
+                        group_id,
+                        label,
+                        pile_column * -22 - front_index * 2,
+                        (len(icons) + 4) // 5,
+                        116,
+                        pile_contents,
+                    ),
+                )
+
         return format_html(
-            '<span class="files-icons files-icons--compact" style="font-size: 1em; opacity: 0.8;">{}</span>',
+            '<span class="files-icons files-icons--compact" style="font-size: 1em;">{}</span>',
             mark_safe("".join(output)),
         )
 
-    @admin.display()
+    @admin.display(empty_value="")
     def size(self, obj):
-        request = self.request
-        config = request.archivebox_config
         archive_size = self._get_progress_stats(obj)["output_size"] or 0
         if archive_size:
-            size_txt = printable_filesize(archive_size)
+            size_txt = _format_size_column(archive_size)
             if archive_size > 52428800:
                 size_txt = mark_safe(f"<b>{size_txt}</b>")
         else:
-            size_txt = mark_safe('<span style="opacity: 0.3">...</span>')
+            return ""
         return format_html(
             '<a href="{}" title="View all files">{}</a>',
-            build_web_url(f"/{obj.archive_path}", request=request, config=config),
+            self.get_snapshot_files_url(obj),
             size_txt,
         )
 
@@ -1111,7 +1273,36 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
         # For started snapshots, show progress bar
         if obj.status == "started" and stats["total"] > 0:
-            return render_snapshot_progress(stats)
+            percent = stats["percent"]
+            running = stats["running"]
+            succeeded = stats["succeeded"]
+            failed = stats["failed"]
+
+            return format_html(
+                """<div style="min-width: 90px;">
+                    <div style="display: flex; align-items: center; gap: 6px; margin-bottom: 4px;">
+                        <span class="snapshot-progress-spinner"></span>
+                        <span style="font-size: 11px; color: #64748b;">{}/{} hooks</span>
+                    </div>
+                    <div style="background: #e2e8f0; border-radius: 4px; height: 6px; overflow: hidden;">
+                        <div style="background: linear-gradient(90deg, #10b981 0%, #10b981 {}%, #ef4444 {}%, #ef4444 {}%, #3b82f6 {}%, #3b82f6 100%);
+                                    width: {}%; height: 100%; transition: width 0.3s;"></div>
+                    </div>
+                    <div style="font-size: 10px; color: #94a3b8; margin-top: 2px;">
+                        ✓{} ✗{} ⏳{}
+                    </div>
+                </div>""",
+                succeeded + failed + stats["skipped"],
+                stats["total"],
+                int(succeeded / stats["total"] * 100) if stats["total"] else 0,
+                int(succeeded / stats["total"] * 100) if stats["total"] else 0,
+                int((succeeded + failed) / stats["total"] * 100) if stats["total"] else 0,
+                int((succeeded + failed) / stats["total"] * 100) if stats["total"] else 0,
+                percent,
+                succeeded,
+                failed,
+                running,
+            )
 
         # For other statuses, show simple badge
         return format_html(
@@ -1125,6 +1316,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     @admin.display(
         description="Size",
         ordering="output_size",
+        empty_value="",
     )
     def size_with_stats(self, obj):
         """Show archive size with output size from archive results."""
@@ -1133,24 +1325,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         size_bytes = output_size or 0
 
         if size_bytes:
-            size_txt = printable_filesize(size_bytes)
+            size_txt = _format_size_column(size_bytes)
             if size_bytes > 52428800:  # 50MB
                 size_txt = mark_safe(f"<b>{size_txt}</b>")
         else:
-            size_txt = mark_safe('<span style="opacity: 0.3">...</span>')
-
-        # Show hook statistics
-        if stats["total"] > 0:
-            return format_html(
-                '<a href="{}" title="View all files" style="white-space: nowrap;">'
-                "{}</a>"
-                '<div style="font-size: 10px; color: #94a3b8; margin-top: 2px;">'
-                "{}/{} hooks</div>",
-                self.get_snapshot_files_url(obj),
-                size_txt,
-                stats["succeeded"],
-                stats["total"],
-            )
+            return ""
 
         return format_html(
             '<a href="{}" title="View all files">{}</a>',
@@ -1159,12 +1338,48 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
 
     def _get_progress_stats(self, obj):
-        if "_admin_progress_stats" not in obj.__dict__:
-            obj._admin_progress_stats = obj.get_progress_stats(
-                results=self._get_prefetched_results(obj),
-                expected_total=self._get_expected_hook_total(obj),
-            )
-        return obj._admin_progress_stats
+        cached_stats = obj.__dict__.get("_admin_progress_stats")
+        if cached_stats is not None:
+            return cached_stats
+
+        results = self._get_prefetched_results(obj)
+        if results is None:
+            stats = obj.get_progress_stats()
+            expected_total = self._get_expected_hook_total(obj)
+            total = max(stats["total"], expected_total)
+            completed = stats["succeeded"] + stats["failed"] + stats.get("skipped", 0) + stats.get("noresults", 0)
+            stats["total"] = total
+            stats["pending"] = max(total - completed - stats["running"], 0)
+            stats["percent"] = int((completed / total * 100) if total > 0 else 0)
+            obj._admin_progress_stats = stats
+            return stats
+
+        expected_total = self._get_expected_hook_total(obj)
+        observed_total = len(results)
+        total = max(observed_total, expected_total)
+        succeeded = sum(1 for r in results if r.status == "succeeded")
+        failed = sum(1 for r in results if r.status == "failed")
+        running = sum(1 for r in results if r.status == "started")
+        skipped = sum(1 for r in results if r.status == "skipped")
+        noresults = sum(1 for r in results if r.status == "noresults")
+        pending = max(total - succeeded - failed - running - skipped - noresults, 0)
+        completed = succeeded + failed + skipped + noresults
+        percent = int((completed / total * 100) if total > 0 else 0)
+        is_sealed = obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED, obj.StatusChoices.PAUSED)
+        stats = {
+            "total": total,
+            "succeeded": succeeded,
+            "failed": failed,
+            "running": running,
+            "pending": pending,
+            "skipped": skipped,
+            "noresults": noresults,
+            "percent": percent,
+            "output_size": obj.output_size or 0,
+            "is_sealed": is_sealed,
+        }
+        obj._admin_progress_stats = stats
+        return stats
 
     def _get_prefetched_results(self, obj):
         if "_admin_output_results" in obj.__dict__:
@@ -1220,17 +1435,39 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             return list(prefetched_cache["tags"])
         return None
 
+    def _get_ordering_fields(self, request):
+        ordering = request.GET.get("o")
+        if not ordering:
+            return set()
+        fields = set()
+        for part in ordering.split("."):
+            if not part:
+                continue
+            try:
+                idx = abs(int(part)) - 1
+            except ValueError:
+                continue
+            if 0 <= idx < len(self.list_display):
+                fields.add(self.list_display[idx])
+        return fields
+
     @admin.display(
         description="Original URL",
         ordering="url",
     )
     def url_str(self, obj):
         return format_html(
-            '<a href="{}"><code style="user-select: all;">{}</code></a>',
+            '<a class="snapshot-original-url" href="{}"><code style="user-select: all;">{}</code></a>',
             obj.url,
-            obj.url[:128],
+            snapshot_url_text(obj.url),
         )
 
+    @admin.display(description="Health", ordering="health")
+    def health_display(self, obj):
+        h = obj.health
+        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
+        return format_html('<span style="color: {};">{}</span>', color, h)
+
     def grid_view(self, request, extra_context=None):
         extra_context = extra_context or {}
         extra_context["snapshot_is_grid_view"] = True
@@ -1334,7 +1571,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
         tags = []
         for name in tag_names:
-            tag, _ = Tag.get_or_create_by_name(
+            tag, _ = get_or_create_tag(
                 name,
                 created_by=request.user if request.user.is_authenticated else None,
             )

archivebox/core/admin_tags.py

@@ -7,12 +7,13 @@ from django.contrib import admin, messages
 from django.contrib.admin.options import IS_POPUP_VAR
 from django.http import HttpRequest, HttpResponseRedirect
 from django.template.response import TemplateResponse
+from django.db.models import Prefetch
 from django.urls import reverse
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 
-from archivebox.base_models.admin import card_fieldset, BaseModelAdmin
-from archivebox.core.models import SnapshotTag, Tag
+from archivebox.base_models.admin import BaseModelAdmin
+from archivebox.core.models import ArchiveResult, SnapshotTag, Tag
 from archivebox.core.tag_util import (
     TAG_HAS_SNAPSHOTS_CHOICES,
     TAG_SORT_CHOICES,
@@ -24,7 +25,8 @@ from archivebox.core.tag_util import (
     normalize_has_snapshots_filter,
     normalize_tag_sort,
 )
-from archivebox.core.routes_util import build_snapshot_url
+from archivebox.core.routes_util import build_snapshot_role_output_url
+from archivebox.plugins.discovery import get_snapshot_role_names
 
 
 class TagInline(admin.TabularInline):
@@ -69,14 +71,44 @@ class TagAdmin(BaseModelAdmin):
     ordering = ["name", "id"]
 
     fieldsets = (
-        card_fieldset("Tag", ("name",)),
-        card_fieldset("Metadata", ("id", "created_by", "created_at", "modified_at")),
-        card_fieldset("Recent Snapshots", ("snapshots",), wide=True),
+        (
+            "Tag",
+            {
+                "fields": ("name",),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Metadata",
+            {
+                "fields": ("id", "created_by", "created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Recent Snapshots",
+            {
+                "fields": ("snapshots",),
+                "classes": ("card", "wide"),
+            },
+        ),
     )
 
     add_fieldsets = (
-        card_fieldset("Tag", ("name",), wide=True),
-        card_fieldset("Metadata", ("created_by",)),
+        (
+            "Tag",
+            {
+                "fields": ("name",),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Metadata",
+            {
+                "fields": ("created_by",),
+                "classes": ("card",),
+            },
+        ),
     )
 
     def get_fieldsets(self, request: HttpRequest, obj: Tag | None = None):
@@ -161,7 +193,26 @@ class TagAdmin(BaseModelAdmin):
 
     @admin.display(description="Snapshots")
     def snapshots(self, tag: Tag):
-        snapshots = tag.snapshot_set.select_related("crawl__created_by").order_by("-downloaded_at", "-created_at", "-pk")[:10]
+        snapshots = (
+            tag.snapshot_set.select_related("crawl__created_by")
+            .prefetch_related(
+                Prefetch(
+                    "archiveresult_set",
+                    queryset=ArchiveResult.objects.filter(
+                        plugin__in=get_snapshot_role_names("list_icon"),
+                        status=ArchiveResult.StatusChoices.SUCCEEDED,
+                    ).only(
+                        "snapshot_id",
+                        "plugin",
+                        "status",
+                        "output_str",
+                        "output_files",
+                    ),
+                    to_attr="_list_icon_results",
+                ),
+            )
+            .order_by("-downloaded_at", "-created_at", "-pk")[:10]
+        )
         total_count = tag.snapshot_set.count()
         if not snapshots:
             return mark_safe(
@@ -184,7 +235,12 @@ class TagAdmin(BaseModelAdmin):
                 </a>
                 """,
                     reverse("admin:core_snapshot_change", args=[snapshot.pk]),
-                    build_snapshot_url(str(snapshot.pk), "favicon.ico"),
+                    build_snapshot_role_output_url(
+                        snapshot,
+                        "list_icon",
+                        archive_results=snapshot._list_icon_results,
+                        fallback_to_default=True,
+                    ),
                     title[:120],
                     snapshot.url[:120],
                 ),

archivebox/core/asgi.py

@@ -7,13 +7,26 @@ For more information on this file, see
 https://docs.djangoproject.com/en/stable/howto/deployment/asgi/
 """
 
-from django.core.asgi import get_asgi_application
+from django.core.handlers.asgi import ASGIHandler
 
 from archivebox.config.django import setup_django
 
 setup_django(check_db=True)
 
-django_application = get_asgi_application()
+
+class ArchiveBoxASGIHandler(ASGIHandler):
+    def create_request(self, scope, body_file):
+        # ASGI already spools the complete body. Django's multipart parser
+        # otherwise treats chunked uploads without Content-Length as empty.
+        if not any(name == b"content-length" for name, _value in scope.get("headers", [])):
+            body_file.seek(0, 2)
+            length = body_file.tell()
+            body_file.seek(0)
+            scope = {**scope, "headers": [*scope.get("headers", []), (b"content-length", str(length).encode("ascii"))]}
+        return super().create_request(scope, body_file)
+
+
+django_application = ArchiveBoxASGIHandler()
 
 
 async def application(scope, receive, send):

archivebox/core/forms.py

@@ -9,7 +9,6 @@ from archivebox.base_models.admin import KeyValueWidget
 from archivebox.config.common import get_config, parse_delete_after
 from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
-from archivebox.crawls.forms import URLFiltersField
 from archivebox.crawls.schedule_util import validate_schedule
 from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
 from archivebox.personas.models import Persona
@@ -91,18 +90,6 @@ def edit_string_for_tag_names(tags) -> str:
     return ", ".join(sorted(names))
 
 
-class FileSizeField(forms.CharField):
-    def to_python(self, value):
-        raw_value = super().to_python(value)
-        try:
-            size = parse_filesize_to_bytes(raw_value) if raw_value else 0
-        except ValueError as err:
-            raise forms.ValidationError(str(err)) from err
-        if size < 0:
-            raise forms.ValidationError(f"{self.label} must be 0 or a positive number of bytes.")
-        return size
-
-
 class AddLinkForm(PluginConfigFormMixin, forms.Form):
     allow_crawl_execution_config_fields = False
 
@@ -149,7 +136,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
-    crawl_max_size = FileSizeField(
+    crawl_max_size = forms.CharField(
         label="Max crawl size",
         required=False,
         initial="0",
@@ -182,7 +169,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
-    snapshot_max_size = FileSizeField(
+    snapshot_max_size = forms.CharField(
         label="Max snapshot size",
         required=False,
         initial="0",
@@ -223,7 +210,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
-    url_filters = URLFiltersField(
+    url_filters = forms.Field(
         label="URL allowlist / denylist",
         required=False,
         widget=URLFiltersWidget(source_selector='textarea[name="url"]'),
@@ -360,7 +347,23 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             cleaned_data["config"] = {}
             return cleaned_data
 
-        cleaned_data["plugins"] = [plugin for _category, field, _title in PLUGIN_GROUPS for plugin in cleaned_data.get(field, [])]
+        # Combine all plugin groups into single list
+        all_selected_plugins = []
+        for field in [
+            "main_plugins",
+            "page_setup_plugins",
+            "media_plugins",
+            "text_plugins",
+            "metadata_plugins",
+            "postprocessing_plugins",
+            "other_plugins",
+        ]:
+            selected = cleaned_data.get(field)
+            if isinstance(selected, list):
+                all_selected_plugins.extend(selected)
+
+        # Store combined list for easy access
+        cleaned_data["plugins"] = all_selected_plugins
 
         plugin_config_overrides = self.clean_plugin_config_overrides(get_config(persona=cleaned_data.get("persona")))
         custom_config = cleaned_data.get("config") or {}
@@ -377,10 +380,34 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             raise forms.ValidationError("Enter at least one valid URL.")
         return value
 
+    def clean_url_filters(self):
+        from archivebox.crawls.models import Crawl
+
+        value = self.cleaned_data.get("url_filters") or {}
+        return {
+            "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
+            "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
+            "same_domain_only": bool(value.get("same_domain_only")),
+            "subpaths_only": bool(value.get("subpaths_only")),
+            "only_new": bool(value.get("only_new")),
+        }
+
     def clean_max_urls(self):
         value = self.cleaned_data.get("max_urls")
         return int(value or 0)
 
+    def clean_crawl_max_size(self):
+        raw_value = str(self.cleaned_data.get("crawl_max_size") or "").strip()
+        if not raw_value:
+            return 0
+        try:
+            value = parse_filesize_to_bytes(raw_value)
+        except ValueError as err:
+            raise forms.ValidationError(str(err))
+        if value < 0:
+            raise forms.ValidationError("Max crawl size must be 0 or a positive number of bytes.")
+        return value
+
     def clean_crawl_timeout(self):
         return self._clean_timeout_seconds(self.cleaned_data.get("crawl_timeout"), "Max crawl time", blank_value=0)
 
@@ -412,6 +439,18 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             raise forms.ValidationError(f"{field_label} must be 0 or greater than 10 seconds.")
         return value
 
+    def clean_snapshot_max_size(self):
+        raw_value = str(self.cleaned_data.get("snapshot_max_size") or "").strip()
+        if not raw_value:
+            return 0
+        try:
+            value = parse_filesize_to_bytes(raw_value)
+        except ValueError as err:
+            raise forms.ValidationError(str(err))
+        if value < 0:
+            raise forms.ValidationError("Max snapshot size must be 0 or a positive number of bytes.")
+        return value
+
     def clean_delete_after(self):
         raw_value = str(self.cleaned_data.get("delete_after") or "0").strip() or "0"
         try:

archivebox/core/middleware.py

@@ -13,6 +13,7 @@ from django.http import HttpResponseForbidden, HttpResponseNotModified
 from django.shortcuts import redirect
 from django.utils import timezone
 from django.utils.http import http_date
+from django.utils.cache import patch_cache_control, patch_vary_headers
 
 from archivebox.config import VERSION
 from archivebox.config.common import get_config, get_request_config
@@ -32,6 +33,7 @@ from archivebox.core.routes_util import (
     split_host_port,
 )
 from archivebox.core.views import OriginalDomainHostView, SnapshotHostView
+from archivebox.misc.serve_static import FAVICON_CACHE_CONTROL
 
 ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in"
 
@@ -123,13 +125,12 @@ def AdminCookieIsolationMiddleware(get_response):
 
 
 def CacheControlMiddleware(get_response):
-    snapshot_path_re = re.compile(r"^/[^/]+/\\d{8}/[^/]+/[0-9a-fA-F-]{8,36}/")
     static_cache_key = (get_COMMIT_HASH() or VERSION or "dev").strip()
 
     def middleware(request):
         response = get_response(request)
 
-        if request.path.startswith("/static/"):
+        if request.path.startswith("/static/") and response.status_code in (200, 304):
             rel_path = request.path[len("/static/") :]
             static_path = finders.find(rel_path)
             if static_path:
@@ -154,15 +155,30 @@ def CacheControlMiddleware(get_response):
                     response.headers["Last-Modified"] = http_date(mtime)
                 return response
 
-        if ("/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path)) and not response.get(
-            "Cache-Control",
-        ):
-            config = request.__dict__.get("archivebox_config")
-            if config is None:
-                config = get_config(resolve_plugins=False)
-                request.archivebox_config = config
-            policy = "private" if config.PERMISSIONS == "private" else "public"
-            response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
+        if response.status_code in (200, 206, 304) and response.get("Cache-Control") == FAVICON_CACHE_CONTROL:
+            # The same raw favicon bytes are shared across authenticated viewers.
+            # Session middleware may have added Cookie to Vary during authorization.
+            vary = [
+                value.strip()
+                for value in response.get("Vary", "").split(",")
+                if value.strip().lower() not in {"", "cookie", "authorization"}
+            ]
+            if vary:
+                response["Vary"] = ", ".join(vary)
+            else:
+                response.headers.pop("Vary", None)
+            return response
+
+        # Other private/unlisted artifacts remain private; auth errors stay unstored.
+        snapshot_policy = getattr(request, "archivebox_cache_policy", None)
+        if snapshot_policy and response.status_code in (200, 206, 304):
+            if not response.get("Cache-Control"):
+                response["Cache-Control"] = f"{snapshot_policy}, max-age=60, stale-while-revalidate=300"
+            if snapshot_policy == "private":
+                patch_cache_control(response, private=True)
+        elif snapshot_policy or not response.get("Cache-Control"):
+            response["Cache-Control"] = "private, no-store"
+        patch_vary_headers(response, ("Cookie", "Authorization"))
         return response
 
     return middleware

archivebox/core/models/__init__.py

@@ -1,17 +0,0 @@
-"""Core persistence models, grouped by entity and queryset responsibility."""
-
-from .archiveresults import ArchiveResult
-from .querysets import SnapshotManager, SnapshotQuerySet, UngroupedSubquery
-from .snapshots import Snapshot, SnapshotMigrationError
-from .tags import SnapshotTag, Tag
-
-__all__ = [
-    "ArchiveResult",
-    "Snapshot",
-    "SnapshotManager",
-    "SnapshotMigrationError",
-    "SnapshotQuerySet",
-    "SnapshotTag",
-    "Tag",
-    "UngroupedSubquery",
-]

archivebox/core/models/querysets.py

@@ -1,361 +0,0 @@
-from __future__ import annotations
-
-from datetime import UTC, datetime
-from typing import Any, ClassVar
-
-from django.core.exceptions import FieldDoesNotExist
-from django.db import models
-
-from archivebox.config import CONSTANTS
-from archivebox.config.common import get_config
-from archivebox.crawls.models import Crawl
-from archivebox.misc.system import atomic_write
-from archivebox.misc.util import (
-    to_json,
-)
-
-
-class UngroupedSubquery(models.Subquery):
-    """Scalar subquery that should not be copied into the outer GROUP BY."""
-
-    def get_group_by_cols(self):
-        return []
-
-
-class SnapshotQuerySet(models.QuerySet):
-    """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
-
-    def bulk_create(self, objs, *args, **kwargs):
-        objs = list(objs)
-        missing_crawl_ids = set()
-        from archivebox.core.permissions import PERMISSIONS_VALUES
-
-        for obj in objs:
-            if isinstance(obj, self.model):
-                config = dict(obj.config or {})
-                permission = str(config.get("PERMISSIONS") or "").strip().lower()
-                if permission not in PERMISSIONS_VALUES and obj.crawl_id:
-                    crawl = getattr(obj, "crawl", None)
-                    if not getattr(crawl, "permissions", None):
-                        missing_crawl_ids.add(str(obj.crawl_id))
-
-        crawl_permissions_by_id = {}
-        if missing_crawl_ids:
-            crawl_permissions_by_id = {
-                str(crawl_id): permissions
-                for crawl_id, permissions in Crawl.objects.filter(pk__in=missing_crawl_ids).values_list("pk", "permissions")
-            }
-
-        from archivebox.misc.db import truncate_overlong_charfields
-
-        for obj in objs:
-            if isinstance(obj, self.model):
-                obj.ensure_permissions_config(crawl_permissions=crawl_permissions_by_id.get(str(obj.crawl_id)))
-                # bulk_create bypasses pre_save, so clamp CharFields here too
-                # (e.g. page titles) to stay within postgres VARCHAR limits.
-                truncate_overlong_charfields(obj)
-        return super().bulk_create(objs, *args, **kwargs)
-
-    def paged_iterator(self, chunk_size: int = 500):
-        """
-        Iterate snapshots using bounded keyset pages instead of one streaming cursor.
-
-        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
-        cursor open until the full queryset is exhausted. That is fine for
-        read-only exports, but update/migration code does filesystem work and
-        writes while iterating; a long-lived read cursor there can stretch lock
-        waits across thousands of rows. This respects the queryset's existing
-        filters, order_by(), select_related(), and prefetch_related() state; if
-        no ordering is defined, it falls back to primary-key order.
-        """
-        pk_field = self.model._meta.pk.name
-        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
-
-        ordering = []
-        for term in raw_ordering:
-            if not isinstance(term, str) or term == "?":
-                ordering = []
-                break
-            field_name = term.removeprefix("-")
-            field_name = pk_field if field_name == "pk" else field_name
-            ordering.append(f"-{field_name}" if term.startswith("-") else field_name)
-
-        ordered_field_names = [term.removeprefix("-") for term in ordering]
-        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
-        try:
-            use_keyset = any(name in unique_field_names for name in ordered_field_names) and not any(
-                self.model._meta.get_field(name).null for name in ordered_field_names
-            )
-        except (AttributeError, FieldDoesNotExist):
-            use_keyset = False
-        # Expressions, nullable/related fields and non-unique orderings retain
-        # Django's ordering through bounded offset pages.
-        if not use_keyset:
-            offset = 0
-            while batch := list(self[offset : offset + chunk_size]):
-                yield from batch
-                offset += chunk_size
-            return
-
-        last_values = None
-        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
-        while True:
-            batch_qs = self.order_by(*ordering)
-            if last_values is not None:
-                page_filter = models.Q()
-                for idx, term in enumerate(ordering):
-                    descending = term.startswith("-")
-                    field_name = term[1:] if descending else term
-                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
-                    comparison = "lt" if descending else "gt"
-                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
-                batch_qs = batch_qs.filter(page_filter)
-
-            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
-            if not batch_rows:
-                break
-
-            pk_idx = value_field_names.index(pk_field)
-            snapshot_ids = [row[pk_idx] for row in batch_rows]
-            snapshots_by_id = {snapshot.pk: snapshot for snapshot in self.filter(pk__in=snapshot_ids).order_by()}
-
-            for row in batch_rows:
-                snapshot_id = row[pk_idx]
-                snapshot = snapshots_by_id.get(snapshot_id)
-                if snapshot is not None:
-                    yield snapshot
-
-            last_values = batch_rows[-1][: len(ordered_field_names)]
-
-    # =========================================================================
-    # Filtering Methods
-    # =========================================================================
-
-    FILTER_TYPES: ClassVar[dict[str, Any]] = {
-        "exact": lambda pattern: models.Q(url=pattern),
-        "substring": lambda pattern: models.Q(url__icontains=pattern),
-        "regex": lambda pattern: models.Q(url__iregex=pattern),
-        "domain": lambda pattern: (
-            models.Q(url__istartswith=f"http://{pattern}")
-            | models.Q(url__istartswith=f"https://{pattern}")
-            | models.Q(url__istartswith=f"ftp://{pattern}")
-        ),
-        "tag": lambda pattern: models.Q(tags__name=pattern),
-        "timestamp": lambda pattern: models.Q(timestamp=pattern),
-    }
-    FILTER_TYPE_CHOICES = tuple(FILTER_TYPES)
-    FILTER_ARG_KEYS = (
-        "after",
-        "before",
-        "filter_type",
-        "filter_patterns",
-        "status",
-        "url__icontains",
-        "url__istartswith",
-        "tag",
-        "crawl_id",
-        "limit",
-        "sort",
-        "search",
-    )
-    SPECIAL_FILTER_ARG_KEYS = frozenset({"filter_patterns", "filter_type", "query", "search", "tag", "before", "after", "limit", "sort"})
-
-    def filter_by_patterns(self, patterns: list[str], filter_type: str = "exact") -> SnapshotQuerySet:
-        """Filter snapshots by URL patterns using specified filter type"""
-        from archivebox.misc.logging import stderr
-
-        q_filter = models.Q()
-        for pattern in patterns:
-            try:
-                q_filter = q_filter | self.FILTER_TYPES[filter_type](pattern)
-            except KeyError:
-                stderr()
-                stderr(f"[X] Got invalid pattern for --filter-type={filter_type}:", color="red")
-                stderr(f"    {pattern}")
-                raise SystemExit(2)
-        return self.filter(q_filter)
-
-    def search(self, **kwargs) -> SnapshotQuerySet:
-
-        from archivebox.core.snapshot_status import filter_snapshots_by_status
-        from archivebox.search.query import apply_snapshot_search
-
-        queryset = self
-        filter_patterns = tuple(str(pattern) for pattern in kwargs.get("filter_patterns") or ())
-        filter_type = kwargs.get("filter_type") or "substring"
-        query = kwargs.get("query")
-        if isinstance(query, (list, tuple)):
-            query = " ".join(str(part) for part in query)
-        query = (query or (" ".join(filter_patterns) if kwargs.get("search") else "")).strip()
-
-        field_names = {field.name for field in self.model._meta.get_fields()}
-        field_names.update(field.attname for field in self.model._meta.fields)
-        field_filters = {
-            key: value
-            for key, value in kwargs.items()
-            if value is not None and key not in self.SPECIAL_FILTER_ARG_KEYS and key.split("__", 1)[0] in field_names
-        }
-        status = field_filters.pop("status", None)
-        queryset = filter_snapshots_by_status(queryset, status)
-        if field_filters:
-            queryset = queryset.filter(**field_filters)
-        if kwargs.get("tag"):
-            queryset = queryset.filter(tags__name__iexact=kwargs["tag"])
-        if kwargs.get("before") is not None:
-            queryset = queryset.filter(bookmarked_at__lt=datetime.fromtimestamp(float(kwargs["before"]), tz=UTC))
-        if kwargs.get("after") is not None:
-            queryset = queryset.filter(bookmarked_at__gt=datetime.fromtimestamp(float(kwargs["after"]), tz=UTC))
-
-        if query:
-            queryset = apply_snapshot_search(
-                queryset,
-                query,
-                search_mode=kwargs.get("search"),
-                ordering=("-created_at",) if not kwargs.get("sort") else None,
-                max_results=kwargs.get("limit"),
-                skip_backend_when_metadata_satisfies_limit=True,
-                include_metadata_for_forced_backend=True,
-            )
-        elif filter_patterns:
-            queryset = queryset.filter_by_patterns(list(filter_patterns), filter_type)
-
-        if kwargs.get("sort"):
-            queryset = queryset.order_by(kwargs["sort"])
-        elif not queryset.query.order_by:
-            queryset = queryset.order_by("-created_at")
-
-        limit = kwargs.get("limit")
-        if limit is not None and limit > 0:
-            queryset = queryset[:limit]
-
-        return queryset
-
-    # =========================================================================
-    # Export Methods
-    # =========================================================================
-
-    def to_json(self, with_headers: bool = False) -> str:
-        """Generate JSON index from snapshots"""
-        import sys
-        from datetime import datetime
-
-        from archivebox.config import VERSION
-
-        config = get_config()
-
-        MAIN_INDEX_HEADER = (
-            {
-                "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
-                "schema": "archivebox.index.json",
-                "copyright_info": config.FOOTER_INFO,
-                "meta": {
-                    "project": "ArchiveBox",
-                    "version": VERSION,
-                    "git_sha": VERSION,
-                    "website": "https://ArchiveBox.io",
-                    "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
-                    "source": "https://github.com/ArchiveBox/ArchiveBox",
-                    "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
-                    "dependencies": {},
-                },
-            }
-            if with_headers
-            else {}
-        )
-
-        snapshot_dicts = [s.to_dict(extended=True, static_export=True) for s in self.iterator(chunk_size=500)]
-
-        if with_headers:
-            output = {
-                **MAIN_INDEX_HEADER,
-                "num_links": len(snapshot_dicts),
-                "updated": datetime.now(UTC),
-                "last_run_cmd": sys.argv,
-                "links": snapshot_dicts,
-            }
-        else:
-            output = snapshot_dicts
-        return to_json(output, indent=4, sort_keys=True)
-
-    def to_csv(self, cols: list[str] | None = None, header: bool = True, separator: str = ",", ljust: int = 0) -> str:
-        """Generate CSV output from snapshots"""
-        cols = cols or ["timestamp", "is_archived", "url"]
-        header_str = separator.join(col.ljust(ljust) for col in cols) if header else ""
-        row_strs = (s.to_csv(cols=cols, ljust=ljust, separator=separator) for s in self.iterator(chunk_size=500))
-        return "\n".join((header_str, *row_strs))
-
-    def to_html(self, with_headers: bool = True) -> str:
-        """Generate main index HTML from snapshots"""
-        from datetime import datetime
-
-        from django.template.loader import render_to_string
-
-        from archivebox.config import VERSION
-        from archivebox.config.version import get_COMMIT_HASH
-
-        config = get_config()
-
-        template = "static_index.html" if with_headers else "minimal_index.html"
-        snapshot_list = list(self.iterator(chunk_size=500))
-        manifest_records = []
-        for snapshot in snapshot_list:
-            outputs = snapshot.discover_outputs(include_filesystem_fallback=True)
-            output_paths = [str(output.get("path") or "") for output in outputs]
-            snapshot._public_preview_paths = [
-                path for preferred in ("screenshot/screenshot.png", "screenshot.png") for path in output_paths if path == preferred
-            ]
-            snapshot._public_favicon_paths = [path for path in output_paths if path in ("favicon/favicon.ico", "favicon.ico")]
-            snapshot.write_html_details()
-            if with_headers:
-                # Use the same portable schema as the JSON export. Rendering
-                # above has already populated result-count caches, archive_size
-                # reuses the sealed output_size field, and tags are prefetched.
-                manifest_records.append(snapshot.to_dict(extended=True, static_export=True))
-
-        if with_headers:
-            manifest = "".join(f"{to_json(record, indent=None, sort_keys=True)}\n" for record in manifest_records)
-            atomic_write(str(CONSTANTS.DATA_DIR / CONSTANTS.JSONL_INDEX_FILENAME), manifest)
-
-        return render_to_string(
-            template,
-            {
-                "version": VERSION,
-                "git_sha": get_COMMIT_HASH() or VERSION,
-                "num_links": str(len(snapshot_list)),
-                "date_updated": datetime.now(UTC).strftime("%Y-%m-%d"),
-                "time_updated": datetime.now(UTC).strftime("%Y-%m-%d %H:%M"),
-                "links": snapshot_list,
-                "FOOTER_INFO": config.FOOTER_INFO,
-                "STATIC_EXPORT": True,
-                "STATIC_EXPORT_DIR": CONSTANTS.DATA_DIR,
-            },
-        )
-
-
-class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ignore[unsupported-base]
-    """Manager for Snapshot model - uses SnapshotQuerySet for chainable methods"""
-
-    def filter(self, *args, **kwargs):
-        domain = kwargs.pop("domain", None)
-        qs = super().filter(*args, **kwargs)
-        if domain:
-            qs = qs.filter(url__icontains=f"://{domain}")
-        return qs
-
-    def get_queryset(self):
-        # Don't prefetch by default - it causes "too many open files" during bulk operations
-        # Views/templates can add .prefetch_related('tags', 'archiveresult_set') where needed
-        return super().get_queryset()
-
-    # =========================================================================
-    # Import Methods
-    # =========================================================================
-
-    def remove(self, atomic: bool = False) -> tuple:
-        """Remove snapshots from the database"""
-        from django.db import transaction
-
-        if atomic:
-            with transaction.atomic():
-                return self.get_queryset().delete()
-        return self.get_queryset().delete()

archivebox/core/models/tags.py

@@ -1,150 +0,0 @@
-from __future__ import annotations
-
-from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, ClassVar
-
-from django.conf import settings
-from django.db import IntegrityError, models
-from django.urls import reverse_lazy
-from django.utils import timezone
-from django.utils.text import slugify
-
-from archivebox.base_models.models import (
-    ModelWithUUID,
-    get_or_create_system_user_pk,
-)
-from archivebox.misc.util import (
-    sanitize_html_text,
-)
-
-if TYPE_CHECKING:
-    from .snapshots import Snapshot
-
-
-class Tag(ModelWithUUID):
-    # Keep AutoField for compatibility with main branch migrations
-    # Don't use UUIDField here - requires complex FK transformation
-    id = models.AutoField(primary_key=True, serialize=False, verbose_name="ID")
-    created_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL,
-        on_delete=models.CASCADE,
-        default=get_or_create_system_user_pk,
-        null=True,
-        related_name="tag_set",
-    )
-    created_at = models.DateTimeField(default=timezone.now, db_index=True, null=True)
-    modified_at = models.DateTimeField(auto_now=True)
-    name = models.CharField(unique=True, blank=False, max_length=100)
-
-    @classmethod
-    def get_or_create_by_name(
-        cls,
-        name: str,
-        *,
-        defaults: Mapping[str, Any] | None = None,
-        created_by=None,
-    ) -> tuple[Tag, bool]:
-        name = cls.normalize_name(name)
-        defaults = dict(defaults or {})
-        if created_by is not None:
-            defaults["created_by"] = created_by
-        tag = cls.objects.filter(name__iexact=name).first()
-        if tag:
-            return tag, False
-        try:
-            return cls.objects.create(name=name, **(defaults or {})), True
-        except IntegrityError:
-            tag = cls.objects.filter(name__iexact=name).first()
-            if tag is None:
-                raise
-            return tag, False
-
-    @staticmethod
-    def normalize_name(name: str) -> str:
-        name = sanitize_html_text(name).strip()
-        if not name:
-            raise ValueError("Tag name is required")
-        return name
-
-    def rename(self, name: str) -> Tag:
-        name = self.normalize_name(name)
-        existing = type(self).objects.filter(name__iexact=name).exclude(pk=self.pk).first()
-        if existing:
-            raise ValueError(f'Tag "{existing.name}" already exists')
-        if self.name != name:
-            self.name = name
-            self.save()
-        return self
-
-    snapshot_set: models.Manager[Snapshot]
-
-    class Meta(ModelWithUUID.Meta):
-        app_label = "core"
-        verbose_name = "Tag"
-        verbose_name_plural = "Tags"
-
-    def __str__(self):
-        return self.name
-
-    def save(self, *args, **kwargs):
-        update_fields = kwargs.get("update_fields")
-        if update_fields is None or "name" in update_fields:
-            self.name = sanitize_html_text(self.name).strip()
-        super().save(*args, **kwargs)
-
-    @property
-    def slug(self) -> str:
-        """ASCII-safe slugified form of the tag name (derived, not stored)."""
-        return slugify(self.name or "") or "tag"
-
-    @property
-    def api_url(self) -> str:
-        return str(reverse_lazy("api-1:get_tag", args=[self.id]))
-
-    def to_json(self) -> dict:
-        """
-        Convert Tag model instance to a JSON-serializable dict.
-        """
-        from archivebox.config import VERSION
-
-        return {
-            "type": "Tag",
-            "schema_version": VERSION,
-            "id": str(self.id),
-            "name": self.name,
-        }
-
-    @staticmethod
-    def from_json(record: dict[str, Any], overrides: dict[str, Any] | None = None):
-        """
-        Create/update Tag from JSON dict.
-
-        Args:
-            record: JSON dict with 'name' field
-            overrides: Optional dict with 'snapshot' to auto-attach tag
-
-        Returns:
-            Tag instance or None
-        """
-        name = record.get("name")
-        if not name:
-            return None
-
-        tag, _ = Tag.get_or_create_by_name(name)
-
-        # Auto-attach to snapshot if in overrides
-        if overrides and "snapshot" in overrides and tag:
-            overrides["snapshot"].add_tag_ids([tag.pk])
-
-        return tag
-
-
-class SnapshotTag(models.Model):
-    id = models.AutoField(primary_key=True)
-    snapshot = models.ForeignKey("Snapshot", db_column="snapshot_id", on_delete=models.CASCADE, to_field="id")
-    tag = models.ForeignKey(Tag, db_column="tag_id", on_delete=models.CASCADE, to_field="id")
-
-    class Meta:
-        app_label = "core"
-        db_table = "core_snapshot_tags"
-        unique_together: ClassVar[list[tuple[str, str]]] = [("snapshot", "tag")]

archivebox/core/preview_util.py

@@ -1,4 +0,0 @@
-__package__ = "archivebox.core"
-
-
-EXTENSION_SCREENSHOT_PLUGIN = "chrome_extension_screenshot"

archivebox/core/recovery_util.py

@@ -140,6 +140,13 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
             orphaned_hook_processes = orphaned_hook_processes.filter(snapshot_pwd_filter)
         else:
             orphaned_hook_processes = orphaned_hook_processes.none()
+    else:
+        # Global startup recovery must stay database-only. Process.pwd points
+        # into archive payload storage, which may be a cold FUSE mount; scanning
+        # every historical unlinked hook here blocks the sole scheduler before
+        # it can claim newly queued work. Explicit crawl recovery below remains
+        # available for the work the caller asked to resume.
+        orphaned_hook_processes = orphaned_hook_processes.none()
     for process in orphaned_hook_processes.only(
         "id",
         "pwd",

archivebox/core/routes_util.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import re
 from typing import Any
-from urllib.parse import urlparse
+from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlsplit, urlunsplit
 
 from django.utils.http import url_has_allowed_host_and_scheme
 
@@ -11,6 +11,8 @@ from archivebox.config.common import get_config, get_request_config
 _SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
 _SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P<suffix>[0-9a-fA-F]{12})$")
 _ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api")
+_URL_PATH_SAFE = "/@-._~!$&'()*+,;=%"
+_URL_FRAGMENT_SAFE = "/@-._~!$&'()*+,;="
 
 
 def split_host_port(host: str) -> tuple[str, str | None]:
@@ -67,7 +69,7 @@ def derive_base_url_from_csrf(config: dict[str, Any] | None = None, **config_kwa
     had ``CSRF_TRUSTED_ORIGINS=https://archive.example.com`` set (required for
     admin login to work). On upgrade, ``BASE_URL`` is the new knob — but it
     defaults to empty, and falling through to ``BIND_ADDR`` produces an
-    unreachable URL like ``http://0.0.0.0:8000``. If the user has exactly one
+    unreachable URL like ``http://0.0.0.0:5797``. If the user has exactly one
     CSRF origin we treat it as the implicit BASE_URL so links/redirects keep
     pointing at the public hostname they already configured.
 
@@ -86,11 +88,6 @@ def get_listen_host(config: dict[str, Any] | None = None, **config_kwargs: Any)
     return (config.BIND_ADDR or "").strip()
 
 
-def get_listen_parts(config: dict[str, Any] | None = None, **config_kwargs: Any) -> tuple[str, str | None]:
-    config = config or get_config(**config_kwargs)
-    return split_host_port(get_listen_host(config=config))
-
-
 def _with_port(host: str, port: str | None) -> str:
     return f"{host}:{port}" if port else host
 
@@ -126,7 +123,7 @@ def canonical_base_host_for_request(request_host: str) -> str:
     """Strip role subdomains and remap loopback hostnames to ``archivebox.localhost``.
 
     Used by the banner suggestion and the in-browser pin endpoint: when the
-    user is hitting the server on raw ``localhost:9292`` or ``127.0.0.1:9292``
+    user is hitting the server on raw ``localhost:5797`` or ``127.0.0.1:5797``
     we want to suggest the wildcard-friendly ``archivebox.localhost`` family
     instead, so the eventual pinned ``BASE_URL`` plays nicely with subdomain
     routing without forcing the user to add a /etc/hosts entry.
@@ -139,7 +136,7 @@ def canonical_base_host_for_request(request_host: str) -> str:
 
 def _root_host_from_listen(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
-    listen_host, listen_port = get_listen_parts(config=config)
+    listen_host, listen_port = split_host_port(get_listen_host(config=config))
     root_host = "archivebox.localhost" if _is_local_bind_host(listen_host) else listen_host
     return _with_port(root_host, listen_port) if root_host else ""
 
@@ -373,11 +370,6 @@ def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any]
     )
 
 
-def get_original_base_url(domain: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
-    return _build_url(get_web_base_url(request=request, config=config), f"/original/{domain}")
-
-
 def build_admin_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     return _build_url(get_admin_base_url(request, config=config, **config_kwargs), path)
 
@@ -390,8 +382,103 @@ def build_snapshot_url(snapshot_id: str, path: str = "", request=None, config: d
     return _build_url(get_snapshot_base_url(snapshot_id, request=request, config=config, **config_kwargs), path)
 
 
+def build_snapshot_detail_path(archive_path: str, output_path: str = "", prefix: str = "/") -> str:
+    """Build the live Django snapshot-detail path and optional output anchor."""
+    normalized_archive_path = quote(str(archive_path or "").strip("/"), safe=_URL_PATH_SAFE)
+    detail_path = f"{prefix}{normalized_archive_path}"
+    normalized_output_path = str(output_path or "").lstrip("#/")
+    if not normalized_output_path:
+        return detail_path
+    fragment = quote(normalized_output_path, safe=_URL_FRAGMENT_SAFE)
+    return f"{detail_path}#{fragment}"
+
+
+def build_snapshot_index_path(archive_path: str, output_path: str = "", prefix: str = "/") -> str:
+    """Build a path to the literal exported ``index.html`` snapshot file."""
+    normalized_archive_path = quote(str(archive_path or "").strip("/"), safe=_URL_PATH_SAFE)
+    index_path = f"{prefix}{normalized_archive_path}/index.html"
+    normalized_output_path = str(output_path or "").lstrip("#/")
+    if not normalized_output_path:
+        return index_path
+    fragment = quote(normalized_output_path, safe=_URL_FRAGMENT_SAFE)
+    return f"{index_path}#{fragment}"
+
+
+def get_snapshot_output_anchor(plugin: str, output_path: str) -> str:
+    """Map root-relative legacy outputs to the plugin card that displays them."""
+    normalized_output_path = str(output_path or "").lstrip("/")
+    return normalized_output_path if normalized_output_path.startswith(f"{plugin}/") else plugin
+
+
+def build_snapshot_detail_url(
+    archive_path: str,
+    output_path: str = "",
+    request=None,
+    config: dict[str, Any] | None = None,
+    **config_kwargs: Any,
+) -> str:
+    """Build a request-aware URL for trusted snapshot UI on the web origin."""
+    return build_web_url(
+        build_snapshot_detail_path(archive_path, output_path=output_path),
+        request=request,
+        config=config,
+        **config_kwargs,
+    )
+
+
+def build_snapshot_role_output_url(
+    snapshot,
+    role: str,
+    request=None,
+    config: dict[str, Any] | None = None,
+    archive_results=None,
+    fallback_to_default: bool = False,
+    **config_kwargs: Any,
+) -> str:
+    """Build the first available output URL for a plugin-owned snapshot role."""
+    from archivebox.plugins.discovery import get_plugin_default_output_path, get_snapshot_role_names
+
+    for plugin in get_snapshot_role_names(role):
+        output_path = snapshot.plugin_output_path(plugin, archive_results=archive_results)
+        if not output_path and fallback_to_default:
+            output_path = get_plugin_default_output_path(plugin)
+        if output_path:
+            return build_snapshot_url(str(snapshot.id), output_path, request=request, config=config, **config_kwargs)
+    return ""
+
+
+def build_snapshot_files_url(
+    snapshot_id: str,
+    path: str = "",
+    request=None,
+    config: dict[str, Any] | None = None,
+    **config_kwargs: Any,
+) -> str:
+    """Build a request-aware directory-browser URL on the isolated replay origin."""
+    directory_path = quote(str(path or "").lstrip("/"), safe=_URL_PATH_SAFE)
+    directory_url = build_snapshot_url(snapshot_id, directory_path, request=request, config=config, **config_kwargs)
+    parts = urlsplit(directory_url)
+    normalized_directory_path = f"{parts.path.rstrip('/')}/"
+    directory_url = urlunsplit((parts.scheme, parts.netloc, normalized_directory_path, parts.query, parts.fragment))
+    return _replace_url_query_params(directory_url, files="1")
+
+
+def build_snapshot_zip_url(
+    snapshot_id: str,
+    path: str = "",
+    request=None,
+    config: dict[str, Any] | None = None,
+    **config_kwargs: Any,
+) -> str:
+    """Build a request-aware ZIP download URL for a snapshot directory."""
+    files_url = build_snapshot_files_url(snapshot_id, path, request=request, config=config, **config_kwargs)
+    return _replace_url_query_params(files_url, download="zip")
+
+
 def build_original_url(domain: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_original_base_url(domain, request=request, config=config, **config_kwargs), path)
+    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
+    base_url = _build_url(get_web_base_url(request=request, config=config), f"/original/{domain}")
+    return _build_url(base_url, path)
 
 
 def _build_url(base_url: str, path: str) -> str:
@@ -402,3 +489,11 @@ def _build_url(base_url: str, path: str) -> str:
     if not path:
         return base_url
     return f"{base_url}{path if path.startswith('/') else f'/{path}'}"
+
+
+def _replace_url_query_params(url: str, **params: str) -> str:
+    parts = urlsplit(url)
+    replacement_keys = set(params)
+    query = [(key, value) for key, value in parse_qsl(parts.query, keep_blank_values=True) if key not in replacement_keys]
+    query.extend((key, value) for key, value in params.items())
+    return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment))

archivebox/core/settings.py

@@ -87,6 +87,7 @@ DJANGO_OBJECT_ACTIONS_DEFAULT_HTTP_METHOD = "POST"
 MIDDLEWARE = [
     "archivebox.core.middleware.TimezoneMiddleware",
     "django.middleware.security.SecurityMiddleware",
+    "archivebox.core.middleware.CacheControlMiddleware",
     "archivebox.core.middleware.AdminCookieIsolationMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.common.CommonMiddleware",
@@ -97,7 +98,6 @@ MIDDLEWARE = [
     "archivebox.core.middleware.ServerSecurityModeMiddleware",
     "archivebox.core.middleware.HostRoutingMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
-    "archivebox.core.middleware.CacheControlMiddleware",
     # Additional middlewares from plugins (if any)
 ]
 
@@ -390,7 +390,7 @@ SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
 # get_base_url() also covers deployments that only set CSRF_TRUSTED_ORIGINS (the
 # implicit-BASE_URL fallback used on 0.7.x->0.9.x upgrades), so HTTPS hardening
 # isn't lost until BASE_URL is migrated. An explicit plain-http base (e.g. local
-# http://archivebox.localhost:8000) disables proxy HTTPS handling.
+# http://archivebox.localhost:5797) disables proxy HTTPS handling.
 BASE_URL_IS_HTTPS = get_base_url(config=CONFIG).strip().lower().startswith("https://")
 if BASE_URL_IS_HTTPS or not CONFIG.BASE_URL:
     SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")

archivebox/core/shutdown_util.py

@@ -101,6 +101,7 @@ def foreground_shutdown_signals(
     *,
     first_signal_message: str | None = "\n[🛑] Got {signal_name}, stopping gracefully...\n",
     on_signal: Callable[[signal.Signals], None] | None = None,
+    interrupt_handlers: dict[signal.Signals, Callable[[], None]] | None = None,
     raise_on_first_signal: bool = True,
 ) -> Iterator[ShutdownSignalState]:
     """Install foreground signal handlers that print an immediate exit notice.
@@ -118,10 +119,20 @@ def foreground_shutdown_signals(
 
     state = ShutdownSignalState()
     previous_active_state = _active_shutdown_state
-    previous_handlers = {sig: signal.getsignal(sig) for sig in handled_signals}
+    previous_handlers = {sig: signal.getsignal(sig) for sig in (*handled_signals, *(interrupt_handlers or {}))}
 
     def raise_keyboard_interrupt(signum, _frame):
         sig = signal.Signals(signum)
+        if interrupt_handlers and sig in interrupt_handlers:
+            # Shield resumable terminal intent from sticky shutdown state.
+            # Ctrl+C is delegated to abx-dl's pause/abort controller; SIGUSR1/2
+            # carry prompt answers. Recording any of these as shutdown here
+            # makes a later skip/retry fail raise_if_shutdown_requested(), or
+            # makes a second Ctrl+C hard-exit before hooks can flush output.
+            # SIGTERM/SIGHUP deliberately bypass this shield: ownership takeover
+            # and parent loss must cancel the old worker without asking for input.
+            interrupt_handlers[sig]()
+            return
         already_requested = state.signal_name is not None
         if not already_requested:
             state.signal_name = sig.name
@@ -129,11 +140,9 @@ def foreground_shutdown_signals(
                 os.write(sys.stdout.fileno(), first_signal_message.format(signal_name=state.signal_name).encode())
         if on_signal is not None:
             on_signal(sig)
-        # Foreground `archivebox add` uses the first signal to abort the active
-        # hook through the bus-facing runner code, then reserves the second
-        # signal for hard foreground-command shutdown. Server/update/run and
-        # other non-interactive commands raise immediately so their finally
-        # blocks can stop owned children without prompting.
+        # Actual shutdown requests remain sticky and a second signal forces
+        # exit. Resumable interactive actions were handled above without
+        # marking this context as shutting down.
         if already_requested:
             os.write(sys.stdout.fileno(), f"\n[🛑] Got {sig.name} again, exiting immediately.\n".encode())
             os._exit(130)
@@ -142,7 +151,7 @@ def foreground_shutdown_signals(
 
     try:
         _active_shutdown_state = state
-        for sig in handled_signals:
+        for sig in previous_handlers:
             signal.signal(sig, raise_keyboard_interrupt)
         yield state
     finally:

archivebox/core/tag_util.py

@@ -5,15 +5,17 @@ from collections import defaultdict
 from typing import Any
 from urllib.parse import unquote
 
-from django.db.models import Count, Exists, F, OuterRef, QuerySet
+from django.contrib.auth.models import User
+from django.db.models import Count, Exists, F, OuterRef, Prefetch, QuerySet
 from django.db.models.functions import Lower
 from django.http import HttpRequest
 from django.urls import reverse
 
 from archivebox.config.common import get_config
+from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag, Tag
+from archivebox.core.routes_util import build_snapshot_detail_url, build_snapshot_role_output_url
 from archivebox.misc.util import sanitize_html_text
-from archivebox.core.routes_util import build_snapshot_url, build_web_url
-from archivebox.core.models import Snapshot, SnapshotTag, Tag
+from archivebox.plugins.discovery import get_snapshot_role_names
 
 
 TAG_SNAPSHOT_PREVIEW_LIMIT = 10
@@ -146,6 +148,34 @@ def get_tag_by_ref(tag_ref: str | int) -> Tag:
     return Tag.objects.get(name__iexact=decoded)
 
 
+def get_or_create_tag(name: str, created_by: User | None = None) -> tuple[Tag, bool]:
+    normalized_name = normalize_tag_name(name)
+    if not normalized_name:
+        raise ValueError("Tag name is required")
+
+    defaults = {"created_by": created_by} if created_by is not None else None
+    return Tag.get_or_create_by_name(normalized_name, defaults=defaults)
+
+
+def rename_tag(tag: Tag, name: str) -> Tag:
+    normalized_name = normalize_tag_name(name)
+    if not normalized_name:
+        raise ValueError("Tag name is required")
+
+    existing = Tag.objects.filter(name__iexact=normalized_name).exclude(pk=tag.pk).first()
+    if existing:
+        raise ValueError(f'Tag "{existing.name}" already exists')
+
+    if tag.name != normalized_name:
+        tag.name = normalized_name
+        tag.save()
+    return tag
+
+
+def delete_tag(tag: Tag) -> tuple[int, dict[str, int]]:
+    return tag.delete()
+
+
 def export_tag_urls(tag: Tag) -> str:
     urls = tag.snapshot_set.order_by("-downloaded_at", "-created_at", "-pk").values_list("url", flat=True)
     return "\n".join(urls)
@@ -173,9 +203,16 @@ def _build_snapshot_preview(snapshot: Snapshot, request: HttpRequest | None = No
         "id": str(snapshot.pk),
         "title": _display_snapshot_title(snapshot),
         "url": snapshot.url,
-        "favicon_url": build_snapshot_url(str(snapshot.pk), "favicon.ico", request=request, config=config),
+        "favicon_url": build_snapshot_role_output_url(
+            snapshot,
+            "list_icon",
+            request=request,
+            config=config,
+            archive_results=snapshot._list_icon_results,
+            fallback_to_default=True,
+        ),
         "admin_url": reverse("admin:core_snapshot_change", args=[snapshot.pk]),
-        "archive_url": build_web_url(f"/{snapshot.archive_path_from_db}/index.html", request=request, config=config),
+        "archive_url": build_snapshot_detail_url(snapshot.archive_path_from_db, request=request, config=config),
         "downloaded_at": snapshot.downloaded_at.isoformat() if snapshot.downloaded_at else None,
     }
 
@@ -192,6 +229,22 @@ def _build_snapshot_preview_map(
     snapshot_tags = (
         SnapshotTag.objects.filter(tag_id__in=tag_ids)
         .select_related("snapshot__crawl__created_by")
+        .prefetch_related(
+            Prefetch(
+                "snapshot__archiveresult_set",
+                queryset=ArchiveResult.objects.filter(
+                    plugin__in=get_snapshot_role_names("list_icon"),
+                    status=ArchiveResult.StatusChoices.SUCCEEDED,
+                ).only(
+                    "snapshot_id",
+                    "plugin",
+                    "status",
+                    "output_str",
+                    "output_files",
+                ),
+                to_attr="_list_icon_results",
+            ),
+        )
         .order_by(
             "tag_id",
             F("snapshot__downloaded_at").desc(nulls_last=True),
@@ -221,10 +274,10 @@ def build_tag_card(tag: Tag, snapshot_previews: list[dict[str, Any]] | None = No
         "num_snapshots": count,
         "filter_url": f"/admin/core/snapshot/?tags__id__exact={tag.pk}",
         "edit_url": f"/admin/core/tag/{tag.pk}/change/",
-        "export_urls_url": reverse("api-1:tag_urls_export", args=[tag.pk]),
-        "export_jsonl_url": reverse("api-1:tag_snapshots_export", args=[tag.pk]),
-        "rename_url": reverse("api-1:rename_tag", args=[tag.pk]),
-        "delete_url": reverse("api-1:delete_tag", args=[tag.pk]),
+        "export_urls_url": f"/api/v1/core/tag/{tag.pk}/urls.txt",
+        "export_jsonl_url": f"/api/v1/core/tag/{tag.pk}/snapshots.jsonl",
+        "rename_url": f"/api/v1/core/tag/{tag.pk}/rename",
+        "delete_url": f"/api/v1/core/tag/{tag.pk}/",
         "snapshots": snapshot_previews or [],
     }
 

archivebox/core/takeover_util.py

@@ -76,6 +76,8 @@ def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:
 
 def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
     """Return the newest live local parent allowed to borrow runner/sonic."""
+    from django.db.models import Q
+
     from archivebox.machine.models import Machine, Process
 
     machine = Machine.current()
@@ -83,7 +85,13 @@ def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
         machine=machine,
         status=Process.StatusChoices.RUNNING,
         pwd=str(data_dir),
-        process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ADD, Process.TypeChoices.UPDATE),
+    ).filter(
+        Q(process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ADD, Process.TypeChoices.UPDATE))
+        # A direct `run` owns its own hooks, just as add/update own theirs via
+        # a supervised one-shot worker. Respawning the server daemon must not
+        # steal the gate back from that newer foreground command. Supervised
+        # runner children cannot elect themselves above their parent command.
+        | (Q(process_type=Process.TypeChoices.ORCHESTRATOR) & ~Q(parent__process_type=Process.TypeChoices.SUPERVISORD)),
     )
     if exclude_id is not None:
         qs = qs.exclude(id=exclude_id)
@@ -131,10 +139,20 @@ def ensure_daemon_stack(*, reason: str = ""):
             rprint(f"[yellow][*] Starting daemon stack for {reason}...[/yellow]")
         worker = start_worker(supervisor, sonic_worker)
 
-    from archivebox.search.sonic_daemon import wait_for_sonic_daemon
-
-    wait_for_sonic_daemon(sonic_event)
-    return worker
+    # Supervisord marks this worker RUNNING with startsecs=0, before Sonic has
+    # opened its search socket. A query launched at that point fails even though
+    # the daemon is about to become ready.
+    deadline = time.monotonic() + 30.0
+    while time.monotonic() < deadline:
+        if is_port_listening(sonic_event.host, sonic_event.port):
+            if time.monotonic() < deadline:
+                return worker
+            break
+        worker = get_worker(supervisor, sonic_worker["name"])
+        if not isinstance(worker, dict) or worker.get("statename") not in ("STARTING", "RUNNING"):
+            raise RuntimeError(f"Sonic search backend worker failed to start: {worker}")
+        time.sleep(min(0.1, max(0.0, deadline - time.monotonic())))
+    raise RuntimeError(f"Sonic search backend is not listening at {sonic_event.url}")
 
 
 def live_runner_processes(*, data_dir: str | Path):

archivebox/core/templatetags/core_tags.py

@@ -1,33 +1,77 @@
+import hashlib
 import logging
+import math
 import os
+import re
 from html import unescape
 from pathlib import Path
 from typing import Any
 from urllib.parse import quote, urlparse
 
-from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target
 from django import template
-from django.templatetags.static import static
 from django.utils import timezone
-from django.utils.html import escape
+from django.utils.html import escape, format_html
 from django.utils.safestring import mark_safe
 from django.utils.text import Truncator
 
 from archivebox.config import CONSTANTS
 from archivebox.core.routes_util import (
+    build_snapshot_detail_url,
+    build_snapshot_files_url,
+    build_snapshot_role_output_url,
     build_snapshot_url,
+    build_snapshot_zip_url,
     get_admin_base_url,
     get_snapshot_base_url,
     get_web_base_url,
 )
 from archivebox.core.setup_wizard import get_base_url_mismatch_context, get_setup_wizard_context
 from archivebox.plugins.discovery import (
+    get_plugin_catalog,
     get_plugin_icon,
     get_plugin_name,
     get_plugin_template,
+    plugin_card_is_interactive,
+    plugin_uses_snapshot_replay,
 )
 
 register = template.Library()
+register.filter("plugin_card_is_interactive", plugin_card_is_interactive)
+
+# Preview cards are clipped windows onto their content, never scroll areas.
+# Apply inside the document too: parent overflow cannot constrain an iframe.
+CARD_OVERFLOW_STYLE = (
+    "<style>html,body{height:100%!important;min-height:0!important}"
+    "html,body,body *{overflow:clip!important;scrollbar-width:none!important}</style>"
+)
+
+
+@register.filter
+def snapshot_url_text(url):
+    """Emphasize the hostname while preserving the exact selectable URL."""
+    url = str(url or "")
+    match = re.match(r"^(?:[a-zA-Z][a-zA-Z0-9+.-]*:)?//([^/?#]*)", url)
+    if not match:
+        return escape(url)
+    netloc = match[1]
+    authority = netloc.rsplit("@", 1)[-1]
+    start = match.start(1) + len(netloc) - len(authority)
+    host_end = authority.find("]") + 1 if authority.startswith("[") else authority.find(":")
+    hostname = authority[:host_end] if host_end > 0 else authority
+    if hostname.lower().startswith("www."):
+        start += 4
+        hostname = hostname[4:]
+    remainder = url[start + len(hostname) :]
+    suffix = re.search(r"[?&#]", remainder)
+    suffix_start = suffix.start() if suffix else len(remainder)
+    return format_html(
+        '<span class="snapshot-url-prefix">{}</span><span class="snapshot-url-domain">{}</span>'
+        '<span class="snapshot-url-path">{}</span><span class="snapshot-url-suffix">{}</span>',
+        url[:start],
+        hostname,
+        remainder[:suffix_start],
+        remainder[suffix_start:],
+    )
 
 
 @register.simple_tag(takes_context=True)
@@ -47,6 +91,47 @@ _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml",
 _IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
 _STATIC_URL_SAFE = "/@-._~!$&'()*+,;="
 
+_VIDEO_FILE_EXTS = {
+    ".mp4",
+    ".webm",
+    ".mkv",
+    ".avi",
+    ".mov",
+    ".flv",
+    ".wmv",
+    ".m4v",
+    ".mpg",
+    ".mpeg",
+    ".ts",
+    ".m2ts",
+    ".mts",
+    ".3gp",
+    ".3g2",
+    ".ogv",
+}
+
+_AUDIO_FILE_EXTS = {
+    ".mp3",
+    ".m4a",
+    ".aac",
+    ".ogg",
+    ".oga",
+    ".opus",
+    ".wav",
+    ".flac",
+    ".alac",
+    ".aiff",
+    ".wma",
+    ".mka",
+    ".ac3",
+    ".eac3",
+    ".dts",
+}
+
+_MEDIA_FILE_EXTS = _VIDEO_FILE_EXTS | _AUDIO_FILE_EXTS
+_BROWSER_VIDEO_FILE_EXTS = {".mp4", ".webm", ".m4v", ".ogv"}
+_BROWSER_AUDIO_FILE_EXTS = {".mp3", ".m4a", ".aac", ".ogg", ".oga", ".opus", ".wav", ".flac"}
+
 
 def _is_web_subdomain_alias(browser_url: str, configured_base_url: str) -> bool:
     """Hide the informational mismatch banner on the conventional public web alias.
@@ -63,12 +148,123 @@ def _is_web_subdomain_alias(browser_url: str, configured_base_url: str) -> bool:
     )
 
 
+def _normalize_output_files(output_files: Any) -> dict[str, dict[str, Any]]:
+    from abx_dl.output_files import OutputManifest
+
+    return OutputManifest.from_value(output_files).as_mapping()
+
+
 def _snapshot_id(value: Any) -> Any:
     from archivebox.core.models import Snapshot
 
     return value.id if isinstance(value, Snapshot) else value
 
 
+def _coerce_output_file_size(value: Any) -> int | None:
+    try:
+        return max(int(value or 0), 0)
+    except (TypeError, ValueError):
+        return None
+
+
+def _count_media_files(result) -> int:
+    try:
+        output_files = _normalize_output_files(result.output_files or {})
+    except (AttributeError, TypeError, ValueError):
+        output_files = {}
+
+    if output_files:
+        return sum(1 for path in output_files if Path(path).suffix.lower() in _MEDIA_FILE_EXTS)
+
+    try:
+        plugin_dir = Path(result.snapshot_dir) / result.plugin
+    except (AttributeError, TypeError, ValueError):
+        return 0
+
+    if not plugin_dir.exists():
+        return 0
+
+    count = 0
+    scanned = 0
+    max_scan = 500
+    for file_path in plugin_dir.rglob("*"):
+        if scanned >= max_scan:
+            break
+        scanned += 1
+        if not file_path.is_file():
+            continue
+        if file_path.suffix.lower() in _MEDIA_FILE_EXTS:
+            count += 1
+    return count
+
+
+def _list_media_files(result, *, include_filesystem_fallback=True) -> list[dict]:
+    media_files: list[dict] = []
+    try:
+        plugin_dir = Path(result.snapshot_dir) / result.plugin if include_filesystem_fallback else None
+    except (AttributeError, TypeError, ValueError):
+        return media_files
+
+    output_files = _normalize_output_files(result.output_files or {})
+    candidates: list[tuple[Path, int | None]] = []
+    if output_files:
+        for path, metadata in output_files.items():
+            rel_path = Path(path)
+            if rel_path.suffix.lower() in _MEDIA_FILE_EXTS:
+                candidates.append((rel_path, _coerce_output_file_size(metadata.get("size"))))
+
+    if not candidates and plugin_dir is not None and plugin_dir.exists():
+        scanned = 0
+        max_scan = 2000
+        for file_path in plugin_dir.rglob("*"):
+            if scanned >= max_scan:
+                break
+            scanned += 1
+            if not file_path.is_file():
+                continue
+            if file_path.suffix.lower() in _MEDIA_FILE_EXTS:
+                try:
+                    rel_path = file_path.relative_to(plugin_dir)
+                except ValueError:
+                    continue
+                try:
+                    size = file_path.stat().st_size
+                except OSError:
+                    size = None
+                candidates.append((rel_path, size))
+
+    for rel_path, size in candidates:
+        if callable(getattr(result, "output_file_path", None)):
+            href = result.output_file_path(str(rel_path), output_file_map=output_files)
+        else:
+            metadata = output_files.get(str(rel_path)) or {}
+            href = str(rel_path) if metadata.get("root_relative") else str(Path(result.plugin) / rel_path)
+        if not href:
+            continue
+        suffix = rel_path.suffix.lower()
+        media_type = "video" if suffix in _VIDEO_FILE_EXTS else "audio"
+        media_files.append(
+            {
+                "name": rel_path.name,
+                "path": href,
+                "size": size,
+                "media_type": media_type,
+                "is_video": media_type == "video",
+                "is_audio": media_type == "audio",
+                "is_browser_playable": suffix in _BROWSER_VIDEO_FILE_EXTS or suffix in _BROWSER_AUDIO_FILE_EXTS,
+            },
+        )
+
+    media_files.sort(
+        key=lambda item: (
+            0 if item["is_video"] and item["is_browser_playable"] else 1 if item["is_audio"] and item["is_browser_playable"] else 2,
+            -int(item.get("size") or 0),
+            item["name"].lower(),
+        ),
+    )
+    return media_files
+
+
 def _resolve_snapshot_output_file(snapshot_dir: str | Path | None, raw_output_path: str | None) -> Path | None:
     if not snapshot_dir or not raw_output_path or str(raw_output_path).strip() in (".", "/", "./"):
         return None
@@ -140,14 +336,6 @@ def _snapshot_url_for_context(context, snapshot, path: str = "") -> str:
     return f"{base_url.rstrip('/')}/{quoted_path}{suffix}"
 
 
-def _build_snapshot_files_url(snapshot_id: str, request=None, config=None, base_url: str | None = None) -> str:
-    return (
-        f"{base_url.rstrip('/')}/index.jsonl"
-        if base_url
-        else build_snapshot_url(str(snapshot_id), "/?files=1", request=request, config=config)
-    )
-
-
 def _build_snapshot_preview_url(
     snapshot_id: str,
     path: str = "",
@@ -159,7 +347,11 @@ def _build_snapshot_preview_url(
     if path == "about:blank":
         return path
     if _is_root_snapshot_output_path(path):
-        return _build_snapshot_files_url(snapshot_id, request=request, config=config, base_url=base_url)
+        return (
+            f"{base_url.rstrip('/')}/index.jsonl"
+            if base_url
+            else build_snapshot_files_url(str(snapshot_id), request=request, config=config)
+        )
     if base_url:
         path_part, separator, query = str(path).lstrip("/").partition("?")
         url = f"{base_url.rstrip('/')}/{quote(path_part, safe=_STATIC_URL_SAFE)}"
@@ -169,10 +361,10 @@ def _build_snapshot_preview_url(
         url = build_snapshot_url(str(snapshot_id), path, request=request, config=config)
     path_parts = Path(path).parts
     plugin = get_plugin_name(plugin) if plugin else (path_parts[0] if len(path_parts) > 1 else "")
-    has_plugin_preview = bool(plugin and get_plugin_template(plugin, "full", fallback=False))
-    if not (
-        _is_text_preview_path(path) or _is_image_preview_path(path) or has_plugin_preview or is_archivewebpage_replay_target(path or "")
-    ):
+    has_plugin_preview = bool(
+        plugin and (get_plugin_template(plugin, "full", fallback=False) or plugin_uses_snapshot_replay(plugin)),
+    )
+    if not (_is_text_preview_path(path) or _is_image_preview_path(path) or has_plugin_preview):
         return url
     separator = "&" if "?" in url else "?"
     return f"{url}{separator}preview=1"
@@ -183,7 +375,7 @@ def _render_text_preview(plugin: str, icon_html: str, snippet: str) -> str:
     plugin_label = escape(plugin or "")
     escaped = escape(snippet)
     return (
-        f'<div class="thumbnail-text" data-plugin="{plugin_attr}" data-compact="1">'
+        f'<div class="thumbnail-text" data-plugin="{plugin_attr}">'
         f'<div class="thumbnail-text-header">'
         f'<span class="thumbnail-compact-icon">{icon_html}</span>'
         f'<span class="thumbnail-text-title">{plugin_label}</span>'
@@ -198,7 +390,7 @@ def _render_fallback_card(plugin: str, icon_html: str, fallback_label: str) -> s
     plugin_label = escape(plugin or "")
     fallback_attr = escape(fallback_label)
     return (
-        f'<div class="thumbnail-compact" data-plugin="{plugin_attr}" data-compact="1">'
+        f'<div class="thumbnail-compact" data-plugin="{plugin_attr}">'
         f'<span class="thumbnail-compact-icon">{icon_html}</span>'
         f'<span class="thumbnail-compact-label">{plugin_label}</span>'
         f'<span class="thumbnail-compact-meta">{fallback_attr}</span>'
@@ -238,12 +430,29 @@ def index(value, position):
 
 
 @register.filter
-def file_size(num_bytes: float) -> str:
+def file_size(num_bytes: float, decimal_places: int = 1) -> str:
     for count in ["Bytes", "KB", "MB", "GB"]:
         if num_bytes > -1024.0 and num_bytes < 1024.0:
-            return f"{num_bytes:3.1f} {count}"
+            if num_bytes >= 0:
+                precision = 10 ** max(0, int(decimal_places))
+                num_bytes = math.ceil(num_bytes * precision) / precision
+            return f"{num_bytes:.{decimal_places}f} {count}"
         num_bytes /= 1024.0
-    return "{:3.1f} {}".format(num_bytes, "TB")
+    if num_bytes >= 0:
+        precision = 10 ** max(0, int(decimal_places))
+        num_bytes = math.ceil(num_bytes * precision) / precision
+    return f"{num_bytes:.{decimal_places}f} TB"
+
+
+@register.filter
+def grid_file_size(num_bytes: float | None) -> str:
+    if not num_bytes or num_bytes <= 0:
+        return ""
+    if num_bytes < 1024**2:
+        return f"{max(1, num_bytes / 1024):.0f}kb"
+    if num_bytes < 1024**3:
+        return f"{num_bytes / 1024**2:.0f}mb"
+    return f"{num_bytes / 1024**3:.1f}gb"
 
 
 @register.filter
@@ -314,9 +523,9 @@ def system_warnings_banner(context):
     """Render the top-of-page warning banner for one of the conditions below,
     in priority order (highest first):
 
-    1. ``mode="unconfigured"``— ``BASE_URL`` is empty. Security/correctness
-       issue: until it's pinned, generated URLs can echo any Host the client
-       sends, and admin/web/api routing has no canonical anchor.
+    1. ``mode="unconfigured"``— ``BASE_URL`` is empty and the operator has
+       not selected safe single-domain routing. That mode intentionally uses
+       each device's connection address, including LAN and Tailscale IPs.
     2. ``mode="base_url_mismatch"`` — the browser reached ArchiveBox through
        an origin that differs from the configured canonical ``BASE_URL``.
     3. ``mode="unsafe"``      — ``SERVER_SECURITY_MODE`` explicitly enables
@@ -343,7 +552,7 @@ def system_warnings_banner(context):
 
         config = get_config(resolve_plugins=False)
 
-    if not config.BASE_URL:
+    if not config.BASE_URL and config.SERVER_SECURITY_MODE != "safe-onedomain-nojsreplay":
         return get_setup_wizard_context(context.get("request"), config)
     mismatch = get_base_url_mismatch_context(context.get("request"), config)
     if mismatch and not _is_web_subdomain_alias(mismatch["browser_url"], mismatch["configured_base_url"]):
@@ -410,66 +619,140 @@ def snapshot_base_url(context, snapshot) -> str:
     return _snapshot_base_url_for_context(context, snapshot)
 
 
+@register.simple_tag(takes_context=True)
+def snapshot_detail_url(context, snapshot) -> str:
+    """Keep the trusted snapshot detail page on web.*, not the replay host."""
+    if context.get("STATIC_EXPORT"):
+        return _snapshot_url_for_context(context, snapshot, "index.html")
+    return build_snapshot_detail_url(
+        snapshot.archive_path_from_db,
+        request=context.get("request"),
+        config=context.get("CONFIG"),
+    )
+
+
 @register.simple_tag(takes_context=True)
 def snapshot_url(context, snapshot, path: str = "") -> str:
     return _snapshot_url_for_context(context, snapshot, path)
 
 
 @register.simple_tag(takes_context=True)
-def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> str:
-    snapshot_id = str(_snapshot_id(snapshot))
-    url_cache = snapshot.__dict__.setdefault("_snapshot_archiveresult_url_cache", {})
+def snapshot_files_url(context, snapshot, path: str = "") -> str:
+    if context.get("STATIC_EXPORT"):
+        return _snapshot_url_for_context(context, snapshot, "index.jsonl")
+    return build_snapshot_files_url(
+        str(_snapshot_id(snapshot)),
+        path,
+        request=context.get("request"),
+        config=context.get("CONFIG"),
+    )
+
+
+@register.simple_tag(takes_context=True)
+def snapshot_zip_url(context, snapshot, path: str = "") -> str:
+    if context.get("STATIC_EXPORT"):
+        return ""
+    return build_snapshot_zip_url(
+        str(_snapshot_id(snapshot)),
+        path,
+        request=context.get("request"),
+        config=context.get("CONFIG"),
+    )
+
+
+@register.simple_tag(takes_context=True)
+def snapshot_role_output_url(context, snapshot, role: str, fallback_to_default: bool = False) -> str:
+    url_cache = snapshot.__dict__.setdefault("_snapshot_role_output_url_cache", {})
     cache_key = (
-        plugin,
-        filename,
+        role,
         bool(context.get("STATIC_EXPORT")),
         str(context.get("STATIC_EXPORT_DIR") or ""),
+        fallback_to_default,
     )
     if cache_key in url_cache:
         return url_cache[cache_key]
 
-    results = None
-    if "_admin_archiveresults" in snapshot.__dict__:
-        results = snapshot.__dict__["_admin_archiveresults"]
-    elif "archiveresult_set" in snapshot.__dict__.get("_prefetched_objects_cache", {}):
-        results = snapshot.archiveresult_set.all()
-    elif "_snapshot_archiveresult_url_results" in snapshot.__dict__:
-        results = snapshot.__dict__["_snapshot_archiveresult_url_results"]
+    from archivebox.plugins.discovery import get_snapshot_role_names
 
-    if results is None:
-        from archivebox.core.models import ArchiveResult
-
-        results = list(
-            ArchiveResult.objects.filter(
-                snapshot_id=snapshot_id,
-                plugin__in=("screenshot", "chrome_extension_screenshot", "favicon"),
-                status=ArchiveResult.StatusChoices.SUCCEEDED,
-            ).only(
-                "plugin",
-                "status",
-                "output_files",
-            ),
+    if context.get("STATIC_EXPORT"):
+        output_path = None
+        for plugin in get_snapshot_role_names(role):
+            output_path = snapshot.plugin_output_path(plugin)
+            if output_path:
+                break
+        url_cache[cache_key] = _snapshot_url_for_context(context, snapshot, output_path) if output_path else ""
+    else:
+        url_cache[cache_key] = build_snapshot_role_output_url(
+            snapshot,
+            role,
+            request=context.get("request"),
+            config=context.get("CONFIG"),
+            fallback_to_default=fallback_to_default,
         )
-        snapshot.__dict__["_snapshot_archiveresult_url_results"] = results
+    return url_cache[cache_key]
 
-    for result in results:
-        if result.plugin != plugin or result.status != "succeeded":
-            continue
-        output_files = result.output_files or {}
-        file_info = output_files.get(filename)
-        if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
-            continue
-        output_path = filename if file_info.get("root_relative") else f"{plugin}/{filename}"
-        url_cache[cache_key] = _snapshot_url_for_context(context, snapshot, output_path)
-        return url_cache[cache_key]
 
-    url_cache[cache_key] = ""
-    return ""
+@register.simple_tag(takes_context=True)
+def snapshot_result_file_url(context, snapshot, result, path: str) -> str:
+    """Build a saved-file URL from one ArchiveResult declaration."""
+    try:
+        output_path = result.output_file_path(path)
+    except (AttributeError, TypeError, ValueError, OverflowError):
+        return ""
+    if not output_path:
+        return ""
+    return _snapshot_url_for_context(context, snapshot, output_path)
+
+
+@register.simple_tag(takes_context=True)
+def snapshot_thumbnail(context, snapshot, admin=False, width="100px", height="100px"):
+    from django.template.loader import render_to_string
+
+    from archivebox.plugins.output_groups import snapshot_thumbnail_result
+
+    results = snapshot.__dict__.get("_snapshot_card_results")
+    if results is None:
+        results = snapshot.__dict__.get("_admin_output_results")
+    if results is None:
+        results = snapshot.__dict__.get("_admin_archiveresults")
+    if results is None:
+        results = snapshot.__dict__.get("_prefetched_objects_cache", {}).get("archiveresult_set")
+    if results is None:
+        results = snapshot.archiveresult_set.exclude(plugin="").only(
+            "id",
+            "snapshot_id",
+            "plugin",
+            "status",
+            "output_str",
+            "output_files",
+            "output_size",
+        )
+
+    try:
+        host = urlparse(snapshot.url).hostname or "Archive"
+    except ValueError:
+        host = "Archive"
+    host = host.removeprefix("www.")
+    hue = int(hashlib.sha256(host.encode()).hexdigest()[:6], 16) % 360
+    return mark_safe(
+        render_to_string(
+            "core/snapshot_thumbnail.html",
+            {
+                **context.flatten(),
+                "card_result": snapshot_thumbnail_result(results),
+                "host": host,
+                "initial": host[:1].upper(),
+                "hue": hue,
+                "width": width,
+                "height": height,
+            },
+        ),
+    )
 
 
 @register.simple_tag(takes_context=True)
 def snapshot_index_row(context, link) -> str:
-    snapshot_base = _snapshot_base_url_for_context(context, link)
+    detail_url = snapshot_detail_url(context, link)
 
     status = getattr(link, "status", None) or "unknown"
     bookmarked_at = getattr(link, "bookmarked_at", None)
@@ -488,7 +771,6 @@ def snapshot_index_row(context, link) -> str:
 
     url = getattr(link, "url", "") or ""
     title = unescape(getattr(link, "title", "") or "")
-    is_pending = status in {"queued", "started", "backoff"}
     title_text = title or url
     tags_str = link.tags_str() if callable(getattr(link, "tags_str", None)) else getattr(link, "tags_str", "")
     tag_html = "".join(f'<span class="snapshot-tag">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag)
@@ -506,50 +788,14 @@ def snapshot_index_row(context, link) -> str:
     archive_size = int(getattr(link, "archive_size", 0) or 0)
     size_cell = file_size(archive_size) if archive_size else '<span class="empty-value">...</span>'
     output_plural = "" if num_outputs == 1 else "s"
-    files_url = _snapshot_url_for_context(context, link, "index.jsonl") if context.get("STATIC_EXPORT") else f"{snapshot_base}/?files=1"
+    files_url = snapshot_files_url(context, link)
 
-    if is_pending:
-        preview_html = '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress"></span>'
-        if not context.get("STATIC_EXPORT"):
-            preview_html = (
-                '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress">'
-                f'<img src="{escape(static("spinner.gif"))}" alt="" decoding="async" loading="lazy">'
-                "</span>"
-            )
-    elif "_public_preview_paths" in link.__dict__:
-        preview_paths = list(getattr(link, "_public_preview_paths", []) or [])
-        if preview_paths:
-            preview_urls = [_snapshot_url_for_context(context, link, path) for path in preview_paths]
-            preview_html = (
-                f'<img src="{escape(preview_urls[0])}" '
-                f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
-                'onerror="nextPublicSnapshotPreview(this)" class="snapshot-preview screenshot" alt="" decoding="async" loading="lazy">'
-            )
-        else:
-            preview_html = '<span class="snapshot-preview snapshot-preview-empty" aria-label="No preview available"></span>'
-    else:
-        preview_urls = [
-            url
-            for url in (
-                snapshot_archiveresult_url(context, link, "screenshot", "screenshot.png"),
-                snapshot_archiveresult_url(context, link, "chrome_extension_screenshot", "screenshot-1.png"),
-                snapshot_archiveresult_url(context, link, "chrome_extension_screenshot", "screenshot.png"),
-            )
-            if url
-        ]
-        if preview_urls:
-            preview_html = (
-                f'<img src="{escape(preview_urls[0])}" '
-                f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
-                'onerror="nextPublicSnapshotPreview(this)" class="snapshot-preview screenshot" alt="" decoding="async" loading="lazy">'
-            )
-        else:
-            preview_html = '<span class="snapshot-preview snapshot-preview-empty" aria-label="No preview available"></span>'
+    preview_html = snapshot_thumbnail(context, link)
 
-    if "_public_favicon_paths" in link.__dict__:
-        favicon_paths = list(getattr(link, "_public_favicon_paths", []) or [])
-        if favicon_paths:
-            favicon_urls = [_snapshot_url_for_context(context, link, path) for path in favicon_paths]
+    if "_public_list_icon_paths" in link.__dict__:
+        list_icon_paths = list(getattr(link, "_public_list_icon_paths", []) or [])
+        if list_icon_paths:
+            favicon_urls = [_snapshot_url_for_context(context, link, path) for path in list_icon_paths]
             favicon_html = (
                 f'<img src="{escape(favicon_urls[0])}" '
                 f'data-fallbacks="{escape(",".join(favicon_urls[1:]))}" '
@@ -559,7 +805,7 @@ def snapshot_index_row(context, link) -> str:
         else:
             favicon_html = '<span class="link-favicon link-favicon-empty" aria-hidden="true"></span>'
     else:
-        favicon_url = snapshot_archiveresult_url(context, link, "favicon", "favicon.ico")
+        favicon_url = snapshot_role_output_url(context, link, "list_icon")
         favicon_html = (
             (
                 f'<img src="{escape(favicon_url)}" '
@@ -574,27 +820,24 @@ def snapshot_index_row(context, link) -> str:
     html = f"""
 <tr class="snapshot-row status-{escape(status)}">
     <td class="snapshot-time" title="{escape(title_time)}" data-sort="{escape(sort_value)}">
-        <a href="{escape(snapshot_base)}/index.html">
+        <a href="{escape(detail_url)}">
             <span>{escape(date_text)}</span>
             <small>{escape(time_text)}</small>
         </a>
     </td>
     <td class="snapshot-preview-cell">
-        <a href="{escape(snapshot_base)}/index.html" title="Open archived snapshot">
+        <a href="{escape(detail_url)}" title="Open archived snapshot">
             {preview_html}
         </a>
     </td>
     <td class="snapshot-title-cell" title="{escape(title or url)}">
         <div class="snapshot-title-line">
-            <a href="{escape(snapshot_base)}/index.html" class="snapshot-favicon-link" title="Open archived snapshot">
-                {favicon_html}
-            </a>
-            <a href="{escape(snapshot_base)}/index.html" class="snapshot-title">
-                {escape(Truncator(title_text).chars(110))}
+            <a href="{escape(detail_url)}" class="snapshot-title">
+                {escape(Truncator(title_text).chars(110) if title_text != url else url)}
             </a>
         </div>
         <a href="{escape(url)}" class="snapshot-url" title="{escape(url)}" target="_blank" rel="noopener noreferrer">
-            {escape(url)}
+            {favicon_html}<span class="snapshot-url-text">{snapshot_url_text(url)}</span>
         </a>
         <span class="snapshot-mobile-saved">Saved {escape(date_text)} at {escape(time_text)}</span>
     </td>
@@ -666,6 +909,28 @@ def plugin_card(context, result) -> str:
         return ""
 
     plugin = get_plugin_name(result.plugin)
+    if not context.get("STATIC_EXPORT"):
+        # The cover and expanded stack both embed this one card document. Run
+        # plugin code on the snapshot origin so it can read saved files without
+        # exposing raw archives or the admin session to the collection page.
+        card_url = build_snapshot_url(
+            str(_snapshot_id(result.snapshot)),
+            f"_card/{result.id}",
+            request=context.get("request"),
+            config=context.get("CONFIG"),
+        )
+        return mark_safe(
+            f'<iframe src="{escape(card_url)}" title="{escape(plugin)} card" '
+            'loading="lazy" fetchpriority="low" sandbox="allow-scripts allow-same-origin" '
+            'style="width:100% !important;height:100% !important;transform:none !important;border:0"></iframe>',
+        )
+    return render_plugin_card_document(context, result)
+
+
+def render_plugin_card_document(context, result) -> str:
+    """Render one plugin card template for the trusted snapshot-origin frame."""
+    plugin = get_plugin_name(result.plugin)
+    has_card_template = bool(get_plugin_template(plugin, "card", fallback=False))
     template_str = get_plugin_template(plugin, "card")
 
     # Use embed_path() for the display path
@@ -673,8 +938,8 @@ def plugin_card(context, result) -> str:
     output_url = _snapshot_url_for_context(context, result.snapshot, raw_output_path or "")
 
     icon_html = get_plugin_icon(plugin)
-    plugin_lower = (plugin or "").lower()
-    media_files = result.media_files() if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
+    media_files = _list_media_files(result, include_filesystem_fallback=bool(context.get("STATIC_EXPORT")))
+    media_file_count = len(media_files)
     if context.get("STATIC_EXPORT") and media_files:
         media_files = [
             item
@@ -684,6 +949,7 @@ def plugin_card(context, result) -> str:
             and ".." not in media_path.parts
             and (Path(result.snapshot_dir) / media_path).is_file()
         ]
+        media_file_count = len(media_files)
     if media_files:
         for item in media_files:
             path = item.get("path") or ""
@@ -692,30 +958,68 @@ def plugin_card(context, result) -> str:
     output_lower = (raw_output_path or "").lower()
     force_text_preview = output_lower.endswith(_TEXT_PREVIEW_EXTS)
 
+    # Static exports cannot execute live preview routes; keep their text snippets.
+    if context.get("STATIC_EXPORT") and force_text_preview:
+        template_str = None
+
     # Create a mini template and render it with context
     try:
-        if template_str and raw_output_path and str(raw_output_path).strip() not in (".", "/", "./") and not force_text_preview:
+        if (
+            template_str
+            and raw_output_path
+            and str(raw_output_path).strip() not in (".", "/", "./")
+            and (has_card_template or not force_text_preview)
+        ):
             tpl = template.Template(template_str)
             ctx = template.Context(
                 {
+                    **context.flatten(),
                     "result": result,
                     "snapshot": result.snapshot,
                     "output_path": output_url,
                     "output_path_raw": raw_output_path,
                     "plugin": plugin,
                     "plugin_icon": icon_html,
-                    "media_file_count": len(media_files),
+                    "media_file_count": media_file_count,
                     "media_files": media_files,
                 },
             )
             rendered = tpl.render(ctx)
             # Only return non-empty content (strip whitespace to check)
             if rendered.strip():
+                if rendered.lstrip().lower().startswith("<!doctype html"):
+                    rendered = rendered.replace("</head>", CARD_OVERFLOW_STYLE + "</head>", 1)
+                if context.get("STATIC_EXPORT") and rendered.lstrip().lower().startswith("<!doctype html"):
+                    # Static exports have no _card route. Keep the same card
+                    # document in both stack positions, with relative saved-file
+                    # URLs resolved against the exported index page.
+                    return mark_safe(
+                        f'<iframe srcdoc="{escape(rendered)}" title="{escape(plugin)} card" '
+                        'loading="lazy" sandbox="allow-scripts allow-same-origin" '
+                        'style="width:100% !important;height:100% !important;max-width:100% !important;'
+                        'transform:none !important;border:0"></iframe>',
+                    )
                 return mark_safe(rendered)
     except (template.TemplateSyntaxError, AttributeError, TypeError, ValueError):
         rendered = ""
 
-    if force_text_preview:
+    if force_text_preview and not context.get("STATIC_EXPORT"):
+        # Read archived text only when the browser requests its preview, never
+        # while rendering the outer snapshot HTML (payloads may live on B2).
+        preview_url = _build_snapshot_preview_url(
+            str(_snapshot_id(result.snapshot)),
+            raw_output_path,
+            request=context.get("request"),
+            config=context.get("CONFIG"),
+            plugin=plugin,
+        )
+        return mark_safe(
+            f'<iframe src="{escape(preview_url)}" title="{escape(plugin)} preview" '
+            'loading="lazy" fetchpriority="low" sandbox="allow-scripts allow-same-origin" '
+            'style="border:0;pointer-events:none"></iframe>',
+        )
+
+    if force_text_preview and context.get("STATIC_EXPORT"):
         preview = _render_text_file_preview(result.snapshot_dir, raw_output_path, plugin, icon_html)
         if preview:
             return mark_safe(preview)
@@ -768,7 +1072,11 @@ def plugin_full(context, result) -> str:
     output_url = _snapshot_url_for_context(context, result.snapshot, raw_output_path)
 
     try:
-        tpl = template.Template(template_str)
+        catalog = get_plugin_catalog()
+        tpl = template.Engine(
+            dirs=[catalog[plugin].path] if plugin in catalog else [],
+            libraries=template.Engine.get_default().libraries,
+        ).from_string(template_str)
         ctx = template.Context(
             {
                 "result": result,
@@ -776,6 +1084,7 @@ def plugin_full(context, result) -> str:
                 "output_path": output_url,
                 "output_path_raw": raw_output_path,
                 "plugin": plugin,
+                "plugin_icon": get_plugin_icon(plugin),
             },
         )
         rendered = tpl.render(ctx)

archivebox/core/views/__init__.py

@@ -1,39 +0,0 @@
-"""HTTP views grouped by public browsing, ingestion, replay, and configuration."""
-
-from archivebox.config.common import find_config_source as find_config_source
-from archivebox.plugins.views import get_config_definition_link as get_config_definition_link
-
-from .add import AddView as AddView
-from .add import WebAddView as WebAddView
-from .config import live_config_list_view as live_config_list_view
-from .config import live_config_value_view as live_config_value_view
-from .lookup import _files_index_target as _files_index_target
-from .lookup import _find_snapshot_by_ref as _find_snapshot_by_ref
-from .public import HealthCheckView as HealthCheckView
-from .public import HomepageView as HomepageView
-from .public import PublicIndexView as PublicIndexView
-from .replay import OriginalDomainHostView as OriginalDomainHostView
-from .replay import OriginalDomainReplayView as OriginalDomainReplayView
-from .replay import SnapshotHostView as SnapshotHostView
-from .replay import SnapshotPathView as SnapshotPathView
-from .replay import SnapshotReplayView as SnapshotReplayView
-from .replay import SnapshotView as SnapshotView
-from .replay import _latest_response_match as _latest_response_match
-from .replay import _latest_responses_root as _latest_responses_root
-from .replay import _original_request_url as _original_request_url
-from .replay import _plugin_full_preview_response as _plugin_full_preview_response
-from .replay import _resolve_archiveresult_relpath as _resolve_archiveresult_relpath
-from .replay import _safe_archive_relpath as _safe_archive_relpath
-from .replay import _serve_original_domain_replay as _serve_original_domain_replay
-from .replay import _serve_responses_path as _serve_responses_path
-from .replay import _serve_snapshot_replay as _serve_snapshot_replay
-from .replay import _visible_response_snapshots_for_domain as _visible_response_snapshots_for_domain
-from .replay_auth import REPLAY_AUTH_SALT as REPLAY_AUTH_SALT
-from .replay_auth import SnapshotReplayAuthView as SnapshotReplayAuthView
-from .replay_auth import _admin_login_redirect_or_forbidden as _admin_login_redirect_or_forbidden
-from .replay_auth import _clean_replay_next as _clean_replay_next
-from .replay_auth import _has_replay_cookie as _has_replay_cookie
-from .replay_auth import _private_snapshot_auth_redirect as _private_snapshot_auth_redirect
-from .replay_auth import _replay_auth_response as _replay_auth_response
-from .replay_auth import _replay_cookie_name as _replay_cookie_name
-from .replay_auth import _replay_payload_is_valid as _replay_payload_is_valid

archivebox/core/views/add.py

@@ -1,363 +0,0 @@
-import json
-from urllib.parse import quote
-
-from django.contrib import messages
-from django.contrib.auth.mixins import UserPassesTestMixin
-from django.http import Http404, HttpRequest, HttpResponse, QueryDict
-from django.shortcuts import redirect
-from django.utils.decorators import method_decorator
-from django.utils.html import format_html
-from django.views.decorators.csrf import csrf_exempt, csrf_protect
-from django.views.generic import FormView
-
-from archivebox.config import VERSION
-from archivebox.config.common import (
-    get_config,
-    get_request_config,
-    redact_sensitive_config,
-)
-from archivebox.core.forms import AddLinkForm
-from archivebox.core.models import Snapshot
-from archivebox.core.permissions import (
-    PERMISSIONS_PUBLIC,
-    direct_snapshots_queryset,
-    filter_personas_by_permissions,
-)
-from archivebox.core.routes_util import (
-    build_admin_url,
-    build_web_url,
-    get_admin_host,
-    get_web_host,
-    host_matches,
-)
-from archivebox.crawls.models import Crawl
-from archivebox.misc.util import (
-    sanitize_html_text,
-    urldecode,
-    validate_url,
-)
-from archivebox.plugins.forms import get_plugin_config_binary_urls
-
-from .replay import SnapshotView
-
-
-@method_decorator(csrf_exempt, name="dispatch")
-class AddView(UserPassesTestMixin, FormView):
-    template_name = "add.html"
-    form_class = AddLinkForm
-
-    def get_initial(self):
-        """Prefill the AddLinkForm with the 'url' GET parameter"""
-        if self.request.method == "GET":
-            url = self.request.GET.get("url", None)
-            if url:
-                return {"url": url if "://" in url else f"https://{url}"}
-
-        return super().get_initial()
-
-    def get_form_kwargs(self):
-        kwargs = super().get_form_kwargs()
-        kwargs["request"] = self.request
-        return kwargs
-
-    def test_func(self):
-        return get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated
-
-    def post(self, request: HttpRequest, *args: object, **kwargs: object):
-        if request.user.is_authenticated:
-            return csrf_protect(super().post)(request, *args, **kwargs)
-        return super().post(request, *args, **kwargs)
-
-    def _can_override_crawl_config(self) -> bool:
-        user = self.request.user
-        return bool(user.is_authenticated and user.is_active and user.is_superuser)
-
-    def _get_custom_config_overrides(self, form: AddLinkForm) -> dict:
-        custom_config = form.cleaned_data.get("config") or {}
-
-        if not isinstance(custom_config, dict):
-            return {}
-
-        if not self._can_override_crawl_config():
-            return {}
-
-        return {str(key): value for key, value in custom_config.items() if not str(key).endswith("_BINARY")}
-
-    def get_context_data(self, **kwargs):
-        context = super().get_context_data(**kwargs)
-        request_config = get_request_config(self.request, resolve_plugins=True)
-        required_search_plugin = f"search_backend_{request_config.SEARCH_BACKEND_ENGINE}".strip()
-        can_override_crawl_config = self._can_override_crawl_config()
-        public_persona_config_keys = {
-            "CRAWL_MAX_CONCURRENT_SNAPSHOTS",
-            "DELETE_AFTER",
-            "ONLY_NEW",
-            "PERMISSIONS",
-            "TIMEOUT",
-        }
-        persona_queryset = context["form"].fields["persona"].queryset
-        if not can_override_crawl_config:
-            persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
-        persona_config_map = {}
-        for persona in persona_queryset.order_by("name"):
-            effective_config = get_config(persona=persona)
-            effective_config_redacted = redact_sensitive_config(effective_config.model_dump(mode="json"))
-            if can_override_crawl_config:
-                raw_config = redact_sensitive_config(persona.config or {})
-                effective_config_json = effective_config_redacted
-                binary_urls = get_plugin_config_binary_urls(effective_config)
-            else:
-                raw_config = {}
-                effective_config_json = {key: effective_config_redacted.get(key) for key in public_persona_config_keys}
-                binary_urls = {}
-            persona_config_map[persona.name] = {
-                "config": raw_config,
-                "effective_config": effective_config_json,
-                "binary_urls": binary_urls,
-            }
-        recent_personas = list(persona_queryset.order_by("-created_at", "name")[:5])
-        return {
-            **context,
-            "title": "Create Crawl",
-            # We can't just call request.build_absolute_uri in the template, because it would include query parameters
-            "absolute_add_path": self.request.build_absolute_uri(self.request.path),
-            "web_base_url": build_web_url("", request=self.request),
-            "VERSION": VERSION,
-            "FOOTER_INFO": request_config.FOOTER_INFO,
-            "required_search_plugin": required_search_plugin,
-            "persona_config_map_json": json.dumps(persona_config_map, sort_keys=True, default=str),
-            "recent_personas": recent_personas,
-            "can_override_crawl_config": can_override_crawl_config,
-            "stdout": "",
-        }
-
-    def _create_crawl_from_form(self, form, *, created_by_id=None) -> Crawl:
-        from archivebox.cli.archivebox_add import add
-
-        urls_input = form.cleaned_data["url"]
-        urls = urls_input
-        submitted_lines = [line.strip() for line in urls_input.splitlines() if line.strip()]
-        if len(submitted_lines) == 1:
-            try:
-                # A lone URL pasted into /add/ is the same user-facing input as
-                # `archivebox add https://...`: queue that URL directly so a
-                # narrow plugin selection like `wget` can archive it without
-                # also needing parser plugins. Multi-line or formatted text
-                # remains verbatim import content for the internal parser root.
-                urls = [validate_url(submitted_lines[0])]
-            except ValueError:
-                pass
-        print(f"[+] Adding URL: {urls_input}")
-
-        # Extract all form fields
-        tag = form.cleaned_data["tag"]
-        depth = int(form.cleaned_data["depth"])
-        max_urls = int(form.cleaned_data.get("max_urls") or 0)
-        crawl_max_size = int(form.cleaned_data.get("crawl_max_size") or 0)
-        crawl_timeout = int(form.cleaned_data.get("crawl_timeout") or 0)
-        timeout = form.cleaned_data.get("timeout")
-        snapshot_max_size = int(form.cleaned_data.get("snapshot_max_size") or 0)
-        delete_after = str(form.cleaned_data.get("delete_after") or "0").strip() or "0"
-        crawl_max_concurrent_snapshots = int(form.cleaned_data["crawl_max_concurrent_snapshots"])
-        permissions = str(form.cleaned_data.get("permissions") or "public").strip().lower()
-        can_override_crawl_config = self._can_override_crawl_config()
-        plugins = ",".join(form.cleaned_data.get("plugins", [])) if can_override_crawl_config else ""
-        schedule = form.cleaned_data.get("schedule", "").strip() if can_override_crawl_config else ""
-        persona = form.cleaned_data.get("persona")
-        start_paused = form.cleaned_data.get("start_paused", False) if can_override_crawl_config else False
-        notes = form.cleaned_data.get("notes", "")
-        url_filters = form.cleaned_data.get("url_filters") or {}
-        plugin_config = form.cleaned_data.get("plugin_config") or {}
-        if not isinstance(plugin_config, dict):
-            plugin_config = {}
-        if not can_override_crawl_config:
-            plugin_config = {}
-        custom_config = self._get_custom_config_overrides(form)
-        custom_config.pop("DEFAULT_PERSONA", None)
-        custom_config.pop("PERMISSIONS", None)
-        if persona:
-            persona.ensure_dirs()
-
-        if created_by_id is None:
-            if self.request.user.is_authenticated:
-                created_by_id = self.request.user.pk
-            else:
-                from archivebox.base_models.models import get_or_create_system_user_pk
-
-                created_by_id = get_or_create_system_user_pk()
-
-        config = {}
-        effective_config = get_config(persona=persona) if persona else get_config()
-        if delete_after != str(effective_config.DELETE_AFTER):
-            config["DELETE_AFTER"] = delete_after
-        if timeout is not None and int(timeout) != int(effective_config.TIMEOUT):
-            config["TIMEOUT"] = int(timeout)
-        if permissions:
-            config["PERMISSIONS"] = permissions
-
-        config.update(plugin_config)
-        config.update(custom_config)
-        if bool(url_filters.get("only_new")) != bool(effective_config.ONLY_NEW):
-            config["ONLY_NEW"] = bool(url_filters.get("only_new"))
-        crawl, _snapshots = add(
-            urls=urls,
-            depth=depth,
-            max_urls=max_urls,
-            crawl_max_size=crawl_max_size,
-            crawl_timeout=crawl_timeout,
-            snapshot_max_size=snapshot_max_size,
-            crawl_max_concurrent_snapshots=crawl_max_concurrent_snapshots,
-            tag=tag,
-            url_allowlist=url_filters.get("allowlist") or "",
-            url_denylist=url_filters.get("denylist") or "",
-            plugins=plugins,
-            persona=persona.name if persona else "Default",
-            bg=True,
-            created_by_id=created_by_id,
-            config=config,
-        )
-        if notes:
-            crawl.safe_update({"notes": sanitize_html_text(notes)}, refresh=False)
-        if permissions and crawl.config.get("PERMISSIONS") != permissions:
-            next_config = {**crawl.config, "PERMISSIONS": permissions}
-            crawl.safe_update({"config": next_config}, refresh=True)
-        if start_paused:
-            crawl.pause()
-
-        # 3. create a CrawlSchedule if schedule is provided
-        if schedule:
-            from archivebox.crawls.models import CrawlSchedule
-
-            crawl_schedule = CrawlSchedule.objects.create(
-                template=crawl,
-                schedule=schedule,
-                is_enabled=True,
-                config=config,
-                label=crawl.label,
-                notes=f"Auto-created from add page. {notes}".strip(),
-                created_by_id=created_by_id,
-            )
-            crawl.schedule = crawl_schedule
-            crawl.safe_update({"schedule": crawl_schedule}, refresh=False)
-
-        return crawl
-
-    def form_valid(self, form):
-        crawl = self._create_crawl_from_form(form)
-
-        urls = form.cleaned_data["url"]
-        schedule = form.cleaned_data.get("schedule", "").strip()
-        rough_url_count = len([url for url in urls.splitlines() if url.strip()])
-
-        schedule_msg = ""
-        if schedule and crawl.schedule_id:
-            schedule_msg = format_html(" and <a href='{}'>scheduled to repeat {}</a>", crawl.schedule.admin_change_url, schedule)
-
-        messages.success(
-            self.request,
-            format_html(
-                "Created crawl with {} starting URL(s){}. Snapshots will be created and archived in the background. <a href='{}'>View Crawl →</a>",
-                rough_url_count,
-                schedule_msg,
-                crawl.admin_change_url,
-            ),
-        )
-
-        # Orchestrator (managed by supervisord) will pick up the queued crawl
-        return redirect(crawl.admin_change_url)
-
-
-class WebAddView(AddView):
-    def _latest_snapshot_for_url(self, requested_url: str):
-        return (
-            direct_snapshots_queryset(
-                self.request,
-                SnapshotView.find_snapshots_for_url(requested_url),
-            )
-            .order_by("-bookmarked_at", "-created_at", "-timestamp")
-            .first()
-        )
-
-    def _normalize_add_url(self, requested_url: str) -> str:
-        if requested_url.startswith(("http://", "https://")):
-            return requested_url
-        return f"https://{requested_url}"
-
-    def dispatch(self, request, *args, **kwargs):
-        requested_url = urldecode(kwargs.get("url", "") or "")
-        if requested_url:
-            snapshot = self._latest_snapshot_for_url(requested_url)
-            if snapshot:
-                return redirect(f"/{snapshot.url_path}")
-
-        request_host = (request.get_host() or "").lower()
-        request_config = get_request_config(request)
-        web_host = get_web_host(config=request_config, request=request)
-        admin_host = get_admin_host(config=request_config, request=request)
-        is_web_host = host_matches(request_host, web_host)
-        is_admin_host = host_matches(request_host, admin_host)
-        if request.user.is_authenticated and not request_config.PUBLIC_ADD_VIEW and is_web_host and not is_admin_host:
-            return redirect(build_admin_url(request.get_full_path(), request=request))
-
-        if not self.test_func():
-            if is_web_host and not is_admin_host:
-                return redirect(build_admin_url(request.get_full_path(), request=request))
-            if is_admin_host:
-                next_url = quote(request.get_full_path(), safe="/:?=&")
-                return redirect(f"{build_admin_url('/admin/login/', request=request)}?next={next_url}")
-            return HttpResponse(
-                format_html(
-                    (
-                        "<center><br/><br/><br/>"
-                        "No Snapshots match the given url: <code>{}</code><br/><br/><br/>"
-                        'Return to the <a href="/" target="_top">Main Index</a>'
-                        "</center>"
-                    ),
-                    requested_url or "",
-                ),
-                content_type="text/html",
-                status=404,
-            )
-
-        return super().dispatch(request, *args, **kwargs)
-
-    def get(self, request: HttpRequest, *args: object, **kwargs: object):
-        requested_url = urldecode(str(kwargs.get("url") or (args[0] if args else "")))
-        if not requested_url:
-            raise Http404
-
-        snapshot = self._latest_snapshot_for_url(requested_url)
-        if snapshot:
-            return redirect(f"/{snapshot.url_path}")
-
-        add_url = self._normalize_add_url(requested_url)
-        assert self.form_class is not None
-        defaults_form = self.form_class()
-        form_data = QueryDict(mutable=True)
-        form_data.update(
-            {
-                "url": add_url,
-                "depth": defaults_form.fields["depth"].initial or "0",
-                "max_urls": defaults_form.fields["max_urls"].initial or 0,
-                "crawl_max_size": defaults_form.fields["crawl_max_size"].initial or "0",
-                "crawl_timeout": defaults_form.fields["crawl_timeout"].initial or 0,
-                "timeout": defaults_form.fields["timeout"].initial or 0,
-                "snapshot_max_size": defaults_form.fields["snapshot_max_size"].initial or "0",
-                "delete_after": defaults_form.fields["delete_after"].initial or "0",
-                "crawl_max_concurrent_snapshots": defaults_form.fields["crawl_max_concurrent_snapshots"].initial,
-                "persona": defaults_form.fields["persona"].initial or "Default",
-                "permissions": defaults_form.fields["permissions"].initial or "public",
-                "config": "{}",
-            },
-        )
-        if defaults_form.fields["start_paused"].initial:
-            form_data["start_paused"] = "on"
-
-        form = self.form_class(data=form_data)
-        if not form.is_valid():
-            return self.form_invalid(form)
-
-        crawl = self._create_crawl_from_form(form)
-        snapshot = Snapshot.from_json({"url": add_url, "tags": form.cleaned_data.get("tag", "")}, overrides={"crawl": crawl})
-        assert snapshot is not None
-        return redirect(f"/{snapshot.url_path}")

archivebox/core/views/config.py

@@ -1,261 +0,0 @@
-import os
-from typing import cast
-
-from admin_data_views.typing import ItemContext, SectionData, TableContext
-from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view
-from django.http import HttpRequest
-from django.utils.html import format_html, format_html_join
-from django.utils.safestring import mark_safe
-
-from archivebox.config import CONSTANTS, CONSTANTS_CONFIG
-from archivebox.config.common import (
-    PLUGIN_CONFIG_SCHEMAS,
-    SENSITIVE_CONFIG_VALUE_REDACTED,
-    _plugin_config_properties,
-    find_config_default,
-    find_config_section,
-    find_config_source,
-    find_config_type,
-    get_all_configs,
-    get_config,
-    redact_sensitive_config,
-)
-from archivebox.config.configset import BaseConfigSet
-from archivebox.plugins.views import get_config_definition_link
-
-
-@render_with_table_view
-def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    CONFIGS = get_all_configs()
-
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
-
-    merged_config = get_config(redact_sensitive=True)
-
-    rows = {
-        "Section": [],
-        "Key": [],
-        "Type": [],
-        "Value": [],
-        "Source": [],
-        "Default": [],
-        # "Documentation": [],
-        # "Aliases": [],
-    }
-
-    for section_id, section in reversed(list(CONFIGS.items())):
-        for key in dict(section):
-            rows["Section"].append(section_id)  # section.replace('_', ' ').title().replace(' Config', '')
-            rows["Key"].append(ItemLink(key, key=key))
-            rows["Type"].append(format_html("<code>{}</code>", find_config_type(key)))
-
-            # Use merged config value (includes machine overrides)
-            actual_value = merged_config.get(key, dict(section)[key])
-            rows["Value"].append(format_html("<code>{}</code>", actual_value))
-
-            # Show where the value comes from
-            source = find_config_source(key, merged_config)
-            source_colors = {"Machine": "purple", "Environment": "blue", "File": "green", "Plugin Default": "teal", "Default": "gray"}
-            rows["Source"].append(format_html('<code style="color: {}">{}</code>', source_colors.get(source, "gray"), source))
-
-            rows["Default"].append(
-                format_html(
-                    '<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{}&type=code"><code style="text-decoration: underline">{}</code></a>',
-                    key,
-                    find_config_default(key) or "See here...",
-                ),
-            )
-
-    section = "CONSTANT"
-    for key in CONSTANTS_CONFIG:
-        rows["Section"].append(section)  # section.replace('_', ' ').title().replace(' Config', '')
-        rows["Key"].append(ItemLink(key, key=key))
-        rows["Type"].append(format_html("<code>{}</code>", type(CONSTANTS_CONFIG[key]).__name__))
-        rows["Value"].append(format_html("<code>{}</code>", redact_sensitive_config(CONSTANTS_CONFIG).get(key)))
-        rows["Source"].append(mark_safe('<code style="color: gray">Constant</code>'))
-        rows["Default"].append(
-            format_html(
-                '<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{}&type=code"><code style="text-decoration: underline">{}</code></a>',
-                key,
-                find_config_default(key) or "See here...",
-            ),
-        )
-
-    return TableContext(
-        title="Computed Configuration Values",
-        table=rows,
-    )
-
-
-@render_with_item_view
-def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-    from archivebox.machine.models import Machine
-
-    CONFIGS = get_all_configs()
-
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
-
-    merged_config = get_config(redact_sensitive=True)
-
-    # Determine all sources for this config value
-    sources_info = []
-
-    # Machine config
-    machine = Machine.current()
-    machine_admin_url = machine.admin_change_url
-    if machine.config and key in machine.config:
-        sources_info.append(("Machine", redact_sensitive_config(machine.config).get(key), "purple"))
-
-    # Environment variable
-    if key in os.environ:
-        sources_info.append(("Environment", redact_sensitive_config(os.environ).get(key), "blue"))
-
-    # Config file value
-    if CONSTANTS.CONFIG_FILE.exists():
-        file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
-        if key in file_config:
-            sources_info.append(("File", redact_sensitive_config(file_config).get(key), "green"))
-
-    # Default value
-    default_val = find_config_default(key)
-    if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
-        sources_info.append(("Plugin Default", default_val, "gray"))
-    elif default_val:
-        sources_info.append(("Default", default_val, "gray"))
-
-    # Final computed value
-    config_source = find_config_source(key, merged_config)
-    final_value = merged_config.get(key, CONFIGS.get(key, None))
-    is_redacted = final_value == SENSITIVE_CONFIG_VALUE_REDACTED
-
-    # Build sources display
-    sources_html = format_html_join(
-        mark_safe("<br/>"),
-        '<b style="color: {}">{}:</b> <code>{}</code>',
-        ((color, source, value) for source, value, color in sources_info),
-    )
-
-    aliases = []
-
-    if key in CONSTANTS_CONFIG:
-        section_header = format_html(
-            '[CONSTANTS]   &nbsp; <b><code style="color: lightgray">{}</code></b> &nbsp; <small>(read-only, hardcoded by ArchiveBox)</small>',
-            key,
-        )
-    elif key in merged_config:
-        section_header = format_html(
-            'data / ArchiveBox.conf &nbsp; [{}]  &nbsp; <b><code style="color: lightgray">{}</code></b>',
-            find_config_section(key),
-            key,
-        )
-    else:
-        section_header = format_html(
-            '[DYNAMIC CONFIG]   &nbsp; <b><code style="color: lightgray">{}</code></b> &nbsp; <small>(read-only, calculated at runtime)</small>',
-            key,
-        )
-
-    definition_url, definition_label = get_config_definition_link(key)
-    redacted_message = (
-        mark_safe(
-            '<b style="color: red">Value is redacted for your security. (Passwords, secrets, API tokens, etc. cannot be viewed in the Web UI)</b><br/><br/>',
-        )
-        if is_redacted
-        else ""
-    )
-    default_command_value = val.strip("'") if (val := find_config_default(key)) else str(final_value).strip("'")
-    machine_config_link = (
-        format_html('<br/><a href="{}">→ Edit <code>{}</code> in Machine.config for this server</a>', machine_admin_url, key)
-        if machine_admin_url
-        else ""
-    )
-    machine_config_tip = (
-        format_html(
-            '<br/><b>Tip:</b> To override <code>{}</code> on this machine, <a href="{}">edit the Machine.config field</a> and add:<br/><code>{}</code>',
-            key,
-            machine_admin_url,
-            f'{{"{key}": "your_value_here"}}',
-        )
-        if machine_admin_url and key not in CONSTANTS_CONFIG
-        else ""
-    )
-
-    section_data = cast(
-        SectionData,
-        {
-            "name": section_header,
-            "description": None,
-            "fields": {
-                "Key": key,
-                "Type": find_config_type(key),
-                "Value": final_value,
-                "Currently read from": config_source,
-            },
-            "help_texts": {
-                "Key": format_html(
-                    """
-                <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{}">Documentation</a>  &nbsp;
-                <span style="display: {}">
-                    Aliases: {}
-                </span>
-            """,
-                    key.lower(),
-                    "inline" if aliases else "none",
-                    ", ".join(aliases),
-                ),
-                "Type": format_html(
-                    """
-                <a href="{}" target="_blank" rel="noopener noreferrer">
-                    See full definition in <code>{}</code>...
-                </a>
-            """,
-                    definition_url,
-                    definition_label,
-                ),
-                "Value": format_html(
-                    """
-                {}
-                <br/><hr/><br/>
-                <b>Configuration Sources (highest priority first):</b><br/><br/>
-                {}
-                <br/><br/>
-                <p style="display: {}">
-                    <i>To change this value, edit <code>data/ArchiveBox.conf</code> or run:</i>
-                    <br/><br/>
-                    <code>archivebox config --set {}="{}"</code>
-                </p>
-            """,
-                    redacted_message,
-                    sources_html,
-                    "block" if key in merged_config and key not in CONSTANTS_CONFIG else "none",
-                    key,
-                    default_command_value,
-                ),
-                "Currently read from": format_html(
-                    """
-                The value shown in the "Value" field comes from the <b>{}</b> source.
-                <br/><br/>
-                Priority order (highest to lowest):
-                <ol>
-                    <li><b style="color: purple">Machine</b> - Machine-specific overrides
-                        {}
-                    </li>
-                    <li><b style="color: blue">Environment</b> - process defaults from environment variables</li>
-                    <li><b style="color: green">File</b> - data/ArchiveBox.conf</li>
-                    <li><b style="color: gray">Plugin Default</b> - Default value from plugin config.json</li>
-                    <li><b style="color: gray">Default</b> - Default value from code</li>
-                </ol>
-                {}
-            """,
-                    config_source,
-                    machine_config_link,
-                    machine_config_tip,
-                ),
-            },
-        },
-    )
-
-    return ItemContext(
-        slug=key,
-        title=key,
-        data=[section_data],
-    )

archivebox/core/views/lookup.py

@@ -1,39 +0,0 @@
-from pathlib import Path
-
-from archivebox.core.models import Snapshot
-from archivebox.core.routes_util import (
-    get_snapshot_lookup_key,
-)
-
-
-def _files_index_target(snapshot: Snapshot, archivefile: str | None) -> str:
-    target = archivefile or ""
-    if target == "index.html":
-        target = ""
-    fullpath = Path(snapshot.output_dir) / target
-    if fullpath.is_file():
-        target = str(Path(target).parent)
-        if target == ".":
-            target = ""
-    return target
-
-
-def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
-    lookup = get_snapshot_lookup_key(snapshot_ref)
-    if not lookup:
-        return None
-
-    snapshots = Snapshot.objects.select_related("crawl", "crawl__created_by")
-
-    if len(lookup) == 12 and "-" not in lookup:
-        return snapshots.filter(id__endswith=lookup).order_by("-created_at", "-downloaded_at").first()
-
-    try:
-        return snapshots.get(pk=lookup)
-    except Snapshot.DoesNotExist:
-        try:
-            return snapshots.get(id__startswith=lookup)
-        except Snapshot.DoesNotExist:
-            return None
-        except Snapshot.MultipleObjectsReturned:
-            return snapshots.filter(id__startswith=lookup).first()

archivebox/core/views/public.py

@@ -1,292 +0,0 @@
-from typing import ClassVar
-
-from django.core.paginator import InvalidPage
-from django.db.models import Case, IntegerField, Value, When
-from django.http import Http404, HttpResponse
-from django.shortcuts import redirect
-from django.views import View
-from django.views.generic.list import ListView
-
-from archivebox.config import VERSION
-from archivebox.config.common import (
-    get_request_config,
-)
-from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
-from archivebox.core.permissions import (
-    PERMISSIONS_PRIVATE,
-    PERMISSIONS_PUBLIC,
-    PERMISSIONS_UNLISTED,
-    public_snapshots_queryset,
-)
-from archivebox.core.routes_util import (
-    build_web_url,
-)
-from archivebox.misc.paginators import AcceleratedPaginator
-from archivebox.search.config import (
-    get_search_mode,
-    get_search_mode_backend,
-    get_search_mode_base,
-    get_search_mode_options,
-)
-from archivebox.search.views import get_cached_public_search_state
-
-from .replay_auth import _admin_login_redirect_or_forbidden
-
-
-class HomepageView(View):
-    def get(self, request):
-        request_config = get_request_config(request)
-        if not request_config.BASE_URL:
-            return _admin_login_redirect_or_forbidden(request)
-
-        if request.user.is_authenticated and request_config.CONTROL_PLANE_ENABLED:
-            return redirect("/admin/core/snapshot/")
-
-        if request_config.PUBLIC_INDEX:
-            return redirect("/public")
-
-        return _admin_login_redirect_or_forbidden(request)
-
-
-class PublicIndexView(ListView):
-    template_name = "public_index.html"
-    model = Snapshot
-    ordering: ClassVar[list[str]] = ["-bookmarked_at", "-created_at"]
-    paginator_class = AcceleratedPaginator
-    public_page_scan_chunk_size = 50
-
-    def get_paginate_by(self, queryset):
-        runtime_config = self.__dict__.get("runtime_config")
-        if runtime_config is None:
-            self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
-        return runtime_config.SNAPSHOTS_PER_PAGE
-
-    def _base_public_snapshot_fields(self) -> tuple[str, ...]:
-        return (
-            "id",
-            "created_at",
-            "modified_at",
-            "url",
-            "timestamp",
-            "bookmarked_at",
-            "title",
-            "downloaded_at",
-            "status",
-            "output_size",
-            "permissions",
-        )
-
-    def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot] | None:
-        target_count = page_number * page_size
-        public_snapshots: list[Snapshot] = []
-        scanned = 0
-        chunk_size = max(self.public_page_scan_chunk_size, page_size)
-        ordered_snapshots = Snapshot.objects.order_by(*self.ordering).only(*self._base_public_snapshot_fields())
-
-        while len(public_snapshots) < target_count:
-            chunk = list(ordered_snapshots[scanned : scanned + chunk_size])
-            if not chunk:
-                break
-            scanned += len(chunk)
-            public_snapshots.extend(snapshot for snapshot in chunk if snapshot.permissions == PERMISSIONS_PUBLIC)
-
-        start = (page_number - 1) * page_size
-        return public_snapshots[start:target_count]
-
-    def paginate_queryset(self, queryset, page_size):
-        if self.request.GET.get("q", default="").strip():
-            return super().paginate_queryset(queryset, page_size)
-
-        public_count = self.get_exact_public_snapshot_count()
-        paginator = self.get_paginator(range(public_count), page_size)
-        page_kwarg = self.kwargs.get(self.page_kwarg)
-        page_query = self.request.GET.get(self.page_kwarg)
-        page_number = page_kwarg or page_query or 1
-
-        try:
-            page = paginator.page(page_number)
-        except InvalidPage as err:
-            raise Http404(f"Invalid page ({page_number}): {err}") from err
-
-        object_list = self._ordered_public_page_from_order_index(page_number=page.number, page_size=page_size)
-        page.object_list = object_list
-        return paginator, page, object_list, page.has_other_pages()
-
-    def get_context_data(self, **kwargs):
-        runtime_config = self.__dict__.get("runtime_config")
-        if runtime_config is None:
-            self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
-        search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
-        search_mode_backend = get_search_mode_backend(search_mode, config=runtime_config)
-        query = self.request.GET.get("q", default="").strip()
-        public_search_state = self.__dict__.get("public_search_state")
-        public_search_pending = bool(query and (public_search_state is None or not public_search_state.get("done")))
-        context = {
-            **super().get_context_data(**kwargs),
-            "VERSION": VERSION,
-            "CONFIG": runtime_config,
-            "COMMIT_HASH": runtime_config.COMMIT_HASH,
-            "FOOTER_INFO": runtime_config.FOOTER_INFO,
-            "WEB_BASE_URL": build_web_url(request=self.request, config=runtime_config),
-            "search_mode": search_mode,
-            "search_mode_options": get_search_mode_options(config=runtime_config),
-            "public_search_stream_pending": public_search_pending,
-        }
-        context["show_search_index_hint"] = bool(
-            query
-            and not public_search_pending
-            and get_search_mode_base(search_mode, config=runtime_config) == "deep"
-            and search_mode_backend
-            and context["paginator"].count == 0,
-        )
-        snapshots = list(context.get("object_list") or ())
-        icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots}
-        tag_names_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
-        preview_paths_by_snapshot: dict[str, list[tuple[int, int, str]]] = {str(snapshot.id): [] for snapshot in snapshots}
-        favicon_paths_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
-        progress_by_snapshot: dict[str, dict[str, int]] = {
-            str(snapshot.id): {
-                "total": 0,
-                "succeeded": 0,
-                "failed": 0,
-                "running": 0,
-                "skipped": 0,
-                "noresults": 0,
-            }
-            for snapshot in snapshots
-        }
-        if icons_by_snapshot:
-            for snapshot_id, tag_name in (
-                SnapshotTag.objects.filter(snapshot_id__in=icons_by_snapshot.keys())
-                .order_by("tag__name")
-                .values_list("snapshot_id", "tag__name")
-                .iterator(chunk_size=1000)
-            ):
-                tag_names_by_snapshot[str(snapshot_id)].append(tag_name)
-
-            preview_plugin_order = {
-                "screenshot": 0,
-                "chrome_extension_screenshot": 1,
-            }
-            preview_candidates = {
-                "screenshot": ("screenshot.png",),
-                "chrome_extension_screenshot": ("screenshot-1.png", "screenshot.png"),
-            }
-
-            def result_output_path(result: ArchiveResult, filename: str) -> str | None:
-                output_files = result.output_files or {}
-                file_info = output_files.get(filename)
-                if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
-                    return None
-                if file_info.get("root_relative"):
-                    return filename
-                return f"{result.plugin}/{filename}"
-
-            for snapshot_id, plugin, status in (
-                ArchiveResult.objects.filter(
-                    snapshot_id__in=icons_by_snapshot.keys(),
-                )
-                .exclude(plugin="")
-                .values_list("snapshot_id", "plugin", "status")
-                .iterator(chunk_size=1000)
-            ):
-                snapshot_key = str(snapshot_id)
-                progress = progress_by_snapshot[snapshot_key]
-                progress["total"] += 1
-                if status == ArchiveResult.StatusChoices.SUCCEEDED:
-                    icons_by_snapshot[snapshot_key].add(plugin)
-                    progress["succeeded"] += 1
-                elif status == ArchiveResult.StatusChoices.FAILED:
-                    progress["failed"] += 1
-                elif status == ArchiveResult.StatusChoices.STARTED:
-                    progress["running"] += 1
-                elif status == ArchiveResult.StatusChoices.SKIPPED:
-                    progress["skipped"] += 1
-                elif status == ArchiveResult.StatusChoices.NORESULTS:
-                    progress["noresults"] += 1
-
-            for result in (
-                ArchiveResult.objects.filter(
-                    snapshot_id__in=icons_by_snapshot.keys(),
-                    status=ArchiveResult.StatusChoices.SUCCEEDED,
-                    plugin__in=(*preview_candidates, "favicon"),
-                )
-                .only("snapshot_id", "plugin", "output_files")
-                .iterator(chunk_size=1000)
-            ):
-                snapshot_key = str(result.snapshot_id)
-                if result.plugin in preview_candidates:
-                    plugin_rank = preview_plugin_order[result.plugin]
-                    for filename_rank, filename in enumerate(preview_candidates[result.plugin]):
-                        output_path = result_output_path(result, filename)
-                        if output_path:
-                            preview_paths_by_snapshot[snapshot_key].append((plugin_rank, filename_rank, output_path))
-                elif result.plugin == "favicon":
-                    output_path = result_output_path(result, "favicon.ico")
-                    if output_path:
-                        favicon_paths_by_snapshot[snapshot_key].append(output_path)
-
-        for snapshot in snapshots:
-            snapshot._icons_compact = True
-            snapshot._icons_archive_results = icons_by_snapshot.get(str(snapshot.id), set())
-            snapshot._icons_progress_stats = progress_by_snapshot.get(str(snapshot.id), {})
-            snapshot.num_outputs_cached = snapshot._icons_progress_stats.get("succeeded", 0)
-            snapshot._tags_str_cached = ",".join(tag_names_by_snapshot.get(str(snapshot.id), []))
-            snapshot._public_preview_paths = [
-                output_path for _plugin_rank, _filename_rank, output_path in sorted(preview_paths_by_snapshot.get(str(snapshot.id), []))
-            ]
-            snapshot._public_favicon_paths = favicon_paths_by_snapshot.get(str(snapshot.id), [])
-            snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
-        context["object_list"] = snapshots
-        return context
-
-    def get_exact_public_snapshot_count(self) -> int:
-        hidden_count = Snapshot.objects.filter(permissions=PERMISSIONS_PRIVATE).count()
-        hidden_count += Snapshot.objects.filter(permissions=PERMISSIONS_UNLISTED).count()
-        return Snapshot.objects.count() - hidden_count
-
-    def get_queryset(self, **kwargs):
-        qs = public_snapshots_queryset(super().get_queryset(**kwargs)).only(*self._base_public_snapshot_fields())
-        query = self.request.GET.get("q", default="").strip()
-
-        if not query:
-            return qs
-
-        cached_state = get_cached_public_search_state(self.request)
-        self.public_search_state = cached_state
-        if cached_state is not None:
-            cached_ids = cached_state.get("ids") or []
-            if not cached_ids:
-                return qs.none()
-            search_rank = Case(
-                *(When(pk=snapshot_id, then=Value(index)) for index, snapshot_id in enumerate(cached_ids)),
-                output_field=IntegerField(),
-            )
-            return qs.filter(pk__in=cached_ids).annotate(search_rank=search_rank).order_by("search_rank", *self.ordering)
-
-        return qs.none()
-
-    def get(self, *args, **kwargs):
-        if self.request.user.is_authenticated:
-            return redirect("/admin/core/snapshot/")
-        if get_request_config(self.request).PUBLIC_INDEX:
-            response = super().get(*args, **kwargs)
-            return response
-        else:
-            return _admin_login_redirect_or_forbidden(self.request)
-
-
-class HealthCheckView(View):
-    """
-    A Django view that renders plain text "OK" for service discovery tools
-    """
-
-    def get(self, request):
-        """
-        Handle a GET request
-        """
-        response = HttpResponse("OK", content_type="text/plain", status=200)
-        response["Access-Control-Allow-Origin"] = "*"
-        response["Access-Control-Expose-Headers"] = "X-ArchiveBox-Health"
-        response["X-ArchiveBox-Health"] = "OK"
-        return response

archivebox/core/views/replay.py

@@ -1,838 +0,0 @@
-import posixpath
-from pathlib import Path
-from urllib.parse import quote, urlparse
-
-from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay
-from django import template
-from django.db.models import Q
-from django.http import Http404, HttpRequest, HttpResponse
-from django.shortcuts import redirect, render
-from django.utils.html import format_html
-from django.utils.safestring import mark_safe
-from django.views import View
-
-from archivebox.config.common import (
-    get_request_config,
-)
-from archivebox.core.models import ArchiveResult, Snapshot
-from archivebox.core.permissions import (
-    can_view_snapshot,
-    direct_snapshots_queryset,
-)
-from archivebox.core.routes_util import (
-    build_admin_url,
-    build_snapshot_url,
-    build_web_url,
-    get_snapshot_host,
-    host_matches,
-)
-from archivebox.misc.serve_static import serve_static_with_byterange_support
-from archivebox.misc.util import (
-    base_url,
-    filter_queryset_by_uuid_substring,
-    without_fragment,
-)
-from archivebox.plugins.discovery import get_plugin_name, get_plugin_template
-from archivebox.progressmonitor.views import live_progress_view
-
-from .lookup import _files_index_target, _find_snapshot_by_ref
-from .replay_auth import _has_replay_cookie, _private_snapshot_auth_redirect, _replay_auth_response
-
-
-class SnapshotView(View):
-    # render static html index from filesystem archive/<timestamp>/index.html
-
-    @staticmethod
-    def find_snapshots_for_url(path: str, *, allow_fallback: bool = True):
-        """Return a queryset of snapshots matching a URL-ish path. URL only — never tries ID matching.
-
-        Use ``find_snapshots_for_id`` separately if you also want to match by snapshot UUID.
-        """
-
-        def _fragmentless_url_query(url: str) -> Q:
-            from archivebox.misc.db import is_postgres
-
-            canonical = without_fragment(url)
-            if not is_postgres():
-                # Use a range comparison (url >= 'canonical#' AND url < 'canonical#\U0010ffff')
-                # instead of LIKE/__startswith — SQLite's case-insensitive LIKE bypasses the
-                # url index and forces a full-table scan over ~1M rows (~250ms). The range
-                # form lets SQLite use a MULTI-INDEX OR and stays under 1ms.
-                return Q(url=canonical) | (Q(url__gte=f"{canonical}#") & Q(url__lt=f"{canonical}#\U0010ffff"))
-            # On postgres the range trick is unsafe: linguistic (ICU/libc) collations
-            # don't compare '#'-suffixed strings bytewise, so the range can miss rows.
-            # startswith compiles to LIKE 'prefix%' with wildcards escaped, which is
-            # correct under any collation and uses the url pattern-ops index.
-            return Q(url=canonical) | Q(url__startswith=f"{canonical}#")
-
-        normalized = without_fragment(path)
-        if path.startswith(("http://", "https://")):
-            # exact url match (indexed) — fastest path
-            qs = Snapshot.objects.filter(_fragmentless_url_query(path))
-            if not allow_fallback or qs.exists():
-                return qs
-            normalized = normalized.split("://", 1)[1]
-
-        # try exact match on full url (without scheme)
-        qs = Snapshot.objects.filter(
-            _fragmentless_url_query("http://" + normalized) | _fragmentless_url_query("https://" + normalized),
-        )
-        if qs.exists():
-            return qs
-
-        # fall back to match on exact base_url
-        base = base_url(normalized)
-        qs = Snapshot.objects.filter(
-            _fragmentless_url_query("http://" + base) | _fragmentless_url_query("https://" + base),
-        )
-        if qs.exists():
-            return qs
-
-        # fall back to matching base_url as prefix
-        return Snapshot.objects.filter(Q(url__startswith="http://" + base) | Q(url__startswith="https://" + base))
-
-    @staticmethod
-    def find_snapshots_for_id(slug: str):
-        """Return a queryset of snapshots matching a (possibly truncated) UUID via prefix or suffix.
-
-        Strips non-hex characters from ``slug`` (so input with or without hyphens both work).
-        Requires at least 8 hex chars — shorter inputs return an empty queryset to avoid
-        scanning the entire snapshots table on too-broad matches.
-        """
-        return filter_queryset_by_uuid_substring(Snapshot.objects.all(), slug)
-
-    @staticmethod
-    def render_live_index(request, snapshot):
-        return render(
-            template_name="core/snapshot.html",
-            request=request,
-            context=snapshot.get_html_details_context(request=request),
-        )
-
-    def get(self, request, path):
-        snapshot = None
-
-        try:
-            slug, archivefile = path.split("/", 1)
-        except (IndexError, ValueError):
-            slug, archivefile = path.split("/", 1)[0], "index.html"
-
-        # slug is a timestamp
-        if slug.replace(".", "").isdigit():
-            # missing trailing slash -> redirect to index
-            if "/" not in path:
-                return redirect(f"{path}/index.html")
-
-            try:
-                try:
-                    snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
-                    if not can_view_snapshot(request, snapshot):
-                        return _private_snapshot_auth_redirect(request, snapshot, archivefile or "index.html")
-                    canonical_base = snapshot.url_path
-                    if canonical_base != snapshot.legacy_archive_path:
-                        target_path = f"/{canonical_base}/{archivefile or 'index.html'}"
-                        query = request.META.get("QUERY_STRING")
-                        if query:
-                            target_path = f"{target_path}?{query}"
-                        return redirect(target_path)
-
-                    if request.GET.get("files"):
-                        target_path = _files_index_target(snapshot, archivefile)
-                        response = serve_static_with_byterange_support(
-                            request,
-                            target_path,
-                            document_root=snapshot.output_dir,
-                            show_indexes=True,
-                            is_archive_replay=True,
-                        )
-                    elif archivefile == "index.html":
-                        # if they requested snapshot index, serve live rendered template instead of static html
-                        response = self.render_live_index(request, snapshot)
-                    else:
-                        target = build_snapshot_url(str(snapshot.id), archivefile, request=request)
-                        query = request.META.get("QUERY_STRING")
-                        if query:
-                            target = f"{target}?{query}"
-                        return redirect(target)
-                    response["Link"] = f'<{snapshot.url}>; rel="canonical"'
-                    return response
-                except Snapshot.DoesNotExist:
-                    if Snapshot.objects.filter(timestamp__startswith=slug).exists():
-                        raise Snapshot.MultipleObjectsReturned
-                    else:
-                        raise
-            except Snapshot.DoesNotExist:
-                # Snapshot does not exist
-                return HttpResponse(
-                    format_html(
-                        (
-                            "<center><br/><br/><br/>"
-                            "No Snapshot directories match the given timestamp/ID: <code>{}</code><br/><br/>"
-                            'You can <a href="/add/" target="_top">add a new Snapshot</a>, or return to the <a href="/" target="_top">Main Index</a>'
-                            "</center>"
-                        ),
-                        slug,
-                        path,
-                    ),
-                    content_type="text/html",
-                    status=404,
-                )
-            except Snapshot.MultipleObjectsReturned:
-                snapshot_hrefs = mark_safe("<br/>").join(
-                    format_html(
-                        '{} <a href="/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
-                        snap.bookmarked_at.strftime("%Y-%m-%d %H:%M:%S"),
-                        snap.archive_path,
-                        snap.timestamp,
-                        snap.url,
-                        snap.title_stripped[:64] or "",
-                    )
-                    for snap in direct_snapshots_queryset(request, Snapshot.objects.filter(timestamp__startswith=slug))
-                    .only("url", "timestamp", "title", "bookmarked_at")
-                    .order_by("-bookmarked_at")
-                )
-                return HttpResponse(
-                    format_html(
-                        ("Multiple Snapshots match the given timestamp/ID <code>{}</code><br/><pre>"),
-                        slug,
-                    )
-                    + snapshot_hrefs
-                    + mark_safe('</pre><br/>Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'),
-                    content_type="text/html",
-                    status=404,
-                )
-            except Http404:
-                assert snapshot  # (Snapshot.DoesNotExist is already handled above)
-
-                # Snapshot dir exists but file within does not e.g. 124235.324234/screenshot.png
-                return HttpResponse(
-                    format_html(
-                        """
-                        <html><head>
-                        <title>Snapshot Not Found</title>
-                        </head><body>
-                        <center><br/><br/><br/>
-                        Snapshot <a href="/{}/index.html" target="_top"><b><code>[{}]</code></b></a>: <a href="{}" target="_blank" rel="noreferrer">{}</a><br/>
-                        was queued on {}, but no files have been saved yet in:<br/><b><a href="/{}/" target="_top"><code>{}</code></a><code>/{}</code></b><br/><br/>
-                        It's possible {} during the last capture on {},<br/>or that the archiving process has not completed yet.<br/>
-                        <pre><code># run this cmd to finish/retry archiving this Snapshot</code><br/>
-                        <code style="user-select: all; color: #333">archivebox update -t timestamp {}</code></pre><br/><br/>
-                        <div class="text-align: left; width: 100%; max-width: 400px">
-                        <i><b>Next steps:</i></b><br/>
-                        - list all the <a href="/{}/" target="_top">Snapshot files <code>.*</code></a><br/>
-                        - view the <a href="/{}/index.html" target="_top">Snapshot <code>./index.html</code></a><br/>
-                        - go to the <a href="/admin/core/snapshot/{}/change/" target="_top">Snapshot admin</a> to edit<br/>
-                        - go to the <a href="/admin/core/snapshot/?id__exact={}" target="_top">Snapshot actions</a> to re-archive<br/>
-                        - or return to <a href="/" target="_top">the main index...</a></div>
-                        </center>
-                        </body></html>
-                        """,
-                        snapshot.archive_path,
-                        snapshot.timestamp,
-                        snapshot.url,
-                        snapshot.url,
-                        str(snapshot.bookmarked_at).split(".")[0],
-                        snapshot.archive_path,
-                        snapshot.timestamp,
-                        archivefile if str(archivefile) != "None" else "",
-                        f"the {archivefile} resource could not be fetched"
-                        if str(archivefile) != "None"
-                        else "the original site was not available",
-                        str(snapshot.bookmarked_at).split(".")[0],
-                        snapshot.timestamp,
-                        snapshot.archive_path,
-                        snapshot.archive_path,
-                        snapshot.pk,
-                        snapshot.id,
-                    ),
-                    content_type="text/html",
-                    status=404,
-                )
-
-        # slug is either a URL or a (possibly truncated) snapshot UUID
-        def _resolve_snapshots_for_slug(slug: str):
-            # full URLs go straight to the url-only path (fast, indexed)
-            if "://" in slug:
-                return SnapshotView.find_snapshots_for_url(slug)
-            # short uuid-shaped slugs (>=8 hex chars after stripping non-hex) try id matching first
-            id_qs = SnapshotView.find_snapshots_for_id(slug)
-            if id_qs.exists():
-                return id_qs
-            return SnapshotView.find_snapshots_for_url(slug)
-
-        snapshots = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path))
-        try:
-            if "://" in path:
-                snapshot = snapshots.order_by("-bookmarked_at").first()
-                if snapshot is None:
-                    raise Snapshot.DoesNotExist
-            else:
-                snapshot = snapshots.get()
-        except Snapshot.DoesNotExist:
-            return HttpResponse(
-                format_html(
-                    (
-                        "<center><br/><br/><br/>"
-                        "No Snapshots match the given url: <code>{}</code><br/><br/><br/>"
-                        'Return to the <a href="/" target="_top">Main Index</a>, or:<br/><br/>'
-                        '+ <i><a href="/add/?url={}" target="_top">Add a new Snapshot for <code>{}</code></a><br/><br/></i>'
-                        "</center>"
-                    ),
-                    base_url(path),
-                    path if "://" in path else f"https://{path}",
-                    path,
-                ),
-                content_type="text/html",
-                status=404,
-            )
-        except Snapshot.MultipleObjectsReturned:
-            snapshot_hrefs = mark_safe("<br/>").join(
-                format_html(
-                    '{} <code style="font-size: 0.8em">{}</code> <a href="/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
-                    snap.bookmarked_at.strftime("%Y-%m-%d %H:%M:%S"),
-                    str(snap.id)[:8],
-                    snap.archive_path,
-                    snap.timestamp,
-                    snap.url,
-                    snap.title_stripped[:64] or "",
-                )
-                for snap in snapshots.only("url", "timestamp", "title", "bookmarked_at").order_by("-bookmarked_at")
-            )
-            return HttpResponse(
-                format_html(
-                    ("Multiple Snapshots match the given URL <code>{}</code><br/><pre>"),
-                    base_url(path),
-                )
-                + snapshot_hrefs
-                + mark_safe('</pre><br/>Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'),
-                content_type="text/html",
-                status=404,
-            )
-
-        target_path = build_snapshot_url(str(snapshot.id), "index.html", request=request)
-        query = request.META.get("QUERY_STRING")
-        if query:
-            target_path = f"{target_path}?{query}"
-        return redirect(target_path)
-
-
-class SnapshotPathView(View):
-    """Serve snapshots by the new URL scheme: /<username>/<YYYYMMDD>/<domain>/<uuid>/..."""
-
-    def get(
-        self,
-        request,
-        username: str,
-        date: str | None = None,
-        domain: str | None = None,
-        snapshot_id: str | None = None,
-        path: str = "",
-        url: str | None = None,
-    ):
-        if username == "system":
-            return redirect(request.path.replace("/system/", "/web/", 1))
-
-        if date and domain and domain == date:
-            raise Http404
-
-        requested_url = url
-        if not requested_url and domain and domain.startswith(("http://", "https://")):
-            requested_url = domain
-
-        snapshot = None
-        snapshots_qs = direct_snapshots_queryset(request, Snapshot.objects.select_related("crawl", "crawl__created_by"))
-        if snapshot_id:
-            snapshot = _find_snapshot_by_ref(snapshot_id)
-            if snapshot and not can_view_snapshot(request, snapshot):
-                return _private_snapshot_auth_redirect(request, snapshot, path or "index.html")
-        else:
-            # fuzzy lookup by date + domain/url (most recent)
-            username_lookup = "system" if username == "web" else username
-            if requested_url:
-                qs = direct_snapshots_queryset(
-                    request,
-                    SnapshotView.find_snapshots_for_url(requested_url)
-                    .select_related("crawl", "crawl__created_by")
-                    .filter(
-                        crawl__created_by__username=username_lookup,
-                    ),
-                )
-            else:
-                qs = snapshots_qs.filter(crawl__created_by__username=username_lookup)
-
-            if date:
-                try:
-                    if len(date) == 4:
-                        qs = qs.filter(bookmarked_at__year=int(date))
-                    elif len(date) == 6:
-                        qs = qs.filter(bookmarked_at__year=int(date[:4]), bookmarked_at__month=int(date[4:6]))
-                    elif len(date) == 8:
-                        qs = qs.filter(
-                            bookmarked_at__year=int(date[:4]),
-                            bookmarked_at__month=int(date[4:6]),
-                            bookmarked_at__day=int(date[6:8]),
-                        )
-                except ValueError:
-                    pass
-
-            if requested_url:
-                snapshot = qs.order_by("-bookmarked_at", "-created_at", "-timestamp").first()
-            else:
-                requested_domain = domain or ""
-                if requested_domain.startswith(("http://", "https://")):
-                    requested_domain = Snapshot.extract_domain_from_url(requested_domain)
-                else:
-                    requested_domain = Snapshot.extract_domain_from_url(f"https://{requested_domain}")
-
-                # Prefer exact domain matches
-                matches = [
-                    s for s in qs.order_by("-bookmarked_at", "-created_at") if Snapshot.extract_domain_from_url(s.url) == requested_domain
-                ]
-                snapshot = matches[0] if matches else qs.order_by("-bookmarked_at", "-created_at", "-timestamp").first()
-
-        if not snapshot:
-            return HttpResponse(
-                format_html(
-                    (
-                        "<center><br/><br/><br/>"
-                        "No Snapshots match the given id or url: <code>{}</code><br/><br/><br/>"
-                        'Return to the <a href="/" target="_top">Main Index</a>'
-                        "</center>"
-                    ),
-                    snapshot_id or requested_url or domain,
-                ),
-                content_type="text/html",
-                status=404,
-            )
-
-        canonical_base = snapshot.url_path
-        if date:
-            requested_base = f"{username}/{date}/{domain or url or ''}"
-        else:
-            requested_base = f"{username}/{domain or url or ''}"
-        if snapshot_id:
-            requested_base = f"{requested_base}/{snapshot_id}"
-        if canonical_base != requested_base:
-            target = f"/{canonical_base}/{path or 'index.html'}"
-            query = request.META.get("QUERY_STRING")
-            if query:
-                target = f"{target}?{query}"
-            return redirect(target)
-
-        archivefile = path or "index.html"
-        if archivefile != "index.html" and not request.GET.get("files"):
-            target = build_snapshot_url(str(snapshot.id), archivefile, request=request)
-            query = request.META.get("QUERY_STRING")
-            if query:
-                target = f"{target}?{query}"
-            return redirect(target)
-
-        if request.GET.get("files"):
-            target_path = _files_index_target(snapshot, archivefile)
-            return serve_static_with_byterange_support(
-                request,
-                target_path,
-                document_root=snapshot.output_dir,
-                show_indexes=True,
-                is_archive_replay=True,
-            )
-
-        if archivefile == "index.html":
-            return SnapshotView.render_live_index(request, snapshot)
-
-        return serve_static_with_byterange_support(
-            request,
-            archivefile,
-            document_root=snapshot.output_dir,
-            show_indexes=True,
-            is_archive_replay=True,
-        )
-
-
-def _safe_archive_relpath(path: str) -> str | None:
-    if not path:
-        return ""
-    cleaned = posixpath.normpath(path)
-    cleaned = cleaned.lstrip("/")
-    if cleaned.startswith("..") or "/../" in f"/{cleaned}/":
-        return None
-    return cleaned
-
-
-def _resolve_archiveresult_relpath(snapshot: Snapshot, rel_path: str) -> tuple[str, ArchiveResult | None]:
-    """Resolve plugin-relative output paths through ArchiveResult.output_files."""
-    parts = Path(rel_path).parts
-    if len(parts) < 2:
-        return rel_path, None
-
-    plugin = parts[0]
-    plugin_relpath = posixpath.join(*parts[1:])
-    results = list(
-        ArchiveResult.objects.filter(
-            snapshot=snapshot,
-            plugin=plugin,
-            status=ArchiveResult.StatusChoices.SUCCEEDED,
-        ).only("plugin", "output_files"),
-    )
-    if not results:
-        return rel_path, None
-
-    for result in results:
-        output_files = result.output_files or {}
-        for candidate in (plugin_relpath, rel_path):
-            file_info = output_files.get(candidate)
-            if not isinstance(file_info, dict):
-                continue
-            if file_info.get("root_relative"):
-                return candidate, result
-            return rel_path, result
-
-    return rel_path, results[0]
-
-
-def _plugin_full_preview_response(
-    request: HttpRequest,
-    snapshot: Snapshot,
-    rel_path: str,
-    result: ArchiveResult | None,
-) -> HttpResponse | None:
-    """Render an explicit plugin full template as a trusted preview wrapper."""
-    if not request.GET.get("preview"):
-        return None
-
-    path_parts = Path(rel_path).parts
-    plugin = get_plugin_name(result.plugin) if result else (path_parts[0] if len(path_parts) > 1 else "")
-    if not plugin:
-        return None
-
-    # ReplayWeb.page needs plugin-owned WACZ inspection and service-worker
-    # context, so it remains the one narrow preview exception below.
-    if plugin == "archivewebpage" and archivewebpage_replay.is_replay_target(rel_path):
-        return None
-
-    template_str = get_plugin_template(plugin, "full", fallback=False)
-    if not template_str:
-        return None
-
-    raw_query = request.GET.copy()
-    raw_query.pop("preview", None)
-    output_url = request.path
-    if raw_query:
-        output_url = f"{output_url}?{raw_query.urlencode()}"
-
-    rendered = (
-        template.Engine(debug=False)
-        .from_string(template_str)
-        .render(
-            template.Context(
-                {
-                    "result": result,
-                    "snapshot": snapshot,
-                    "output_path": output_url,
-                    "output_path_raw": rel_path,
-                    "plugin": plugin,
-                    "preview_base": f"{request.path.rsplit('/', 1)[0]}/",
-                },
-            ),
-        )
-    )
-    response = HttpResponse(rendered, content_type="text/html; charset=utf-8")
-    response.headers["Content-Disposition"] = f'inline; filename="{Path(rel_path).stem}.html"'
-    response.headers["X-Content-Type-Options"] = "nosniff"
-    response.headers["X-ArchiveBox-Security-Mode"] = request.archivebox_config.SERVER_SECURITY_MODE
-    response.headers["Referrer-Policy"] = "no-referrer"
-    # Trusted viewers live on the snapshot origin, but the collection UI lives
-    # on web.* (and can be embedded by admin.*). Permit only those configured
-    # origins, not arbitrary sites or other snapshots. This does not grant the
-    # archived document access to the parent frame or its admin session cookies.
-    viewer_origins = " ".join(dict.fromkeys((build_web_url(request=request), build_admin_url(request=request))))
-    response.headers["Content-Security-Policy"] = (
-        "default-src 'self' data: blob:; "
-        "script-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; "
-        "style-src 'unsafe-inline' data: blob: 'self'; "
-        "connect-src 'self' data: blob:; "
-        "img-src 'self' data: blob:; "
-        "media-src 'self' data: blob:; "
-        "font-src 'self' data: blob:; "
-        "frame-src 'self' data: blob:; "
-        "worker-src 'self' blob:; "
-        "object-src 'none'; "
-        "base-uri 'none'; "
-        "form-action 'none'; "
-        f"frame-ancestors 'self' {viewer_origins};"
-    )
-    return response
-
-
-def _visible_response_snapshots_for_domain(request: HttpRequest, domain: str) -> list[Snapshot]:
-    if not domain:
-        return []
-
-    requested_domain = domain.split(":", 1)[0].lower()
-    roots = (f"http://{requested_domain}", f"https://{requested_domain}")
-    domain_query = Q(url__in=roots)
-
-    from archivebox.misc.db import is_postgres
-
-    postgres = is_postgres()
-    for root in roots:
-        for separator in ("/", "?", "#"):
-            prefix = f"{root}{separator}"
-            if postgres:
-                domain_query |= Q(url__startswith=prefix)
-            else:
-                domain_query |= Q(url__gte=prefix, url__lt=f"{prefix}\U0010ffff")
-
-    candidates = (
-        Snapshot.objects.filter(domain_query)
-        .select_related("crawl", "crawl__created_by")
-        .order_by("-bookmarked_at", "-created_at", "-timestamp")
-    )
-    return [snapshot for snapshot in candidates if can_view_snapshot(request, snapshot) or _has_replay_cookie(request, snapshot)]
-
-
-def _latest_response_match(
-    snapshots: list[Snapshot],
-    domain: str,
-    rel_path: str,
-) -> tuple[Snapshot, Path, Path] | None:
-    if not domain or not rel_path:
-        return None
-    requested_domain = domain.split(":", 1)[0].lower()
-    rel_to_root = Path(rel_path)
-    for snapshot in snapshots:
-        responses_root = Path(snapshot.output_dir) / "responses" / requested_domain
-        if (responses_root / rel_to_root).exists():
-            return snapshot, responses_root, rel_to_root
-    return None
-
-
-def _latest_responses_root(snapshots: list[Snapshot], domain: str) -> tuple[Snapshot, Path] | None:
-    if not domain:
-        return None
-    requested_domain = domain.split(":", 1)[0].lower()
-    for snapshot in snapshots:
-        responses_root = Path(snapshot.output_dir) / "responses" / requested_domain
-        if responses_root.is_dir():
-            return snapshot, responses_root
-    return None
-
-
-def _original_request_url(domain: str, path: str = "", query_string: str = "") -> str:
-    normalized_domain = (domain or "").split(":", 1)[0].lower()
-    normalized_path = (path or "").lstrip("/")
-    if normalized_path in ("", "index.html"):
-        normalized_path = ""
-    target = f"https://{normalized_domain}"
-    if normalized_path:
-        target = f"{target}/{normalized_path}"
-    if query_string:
-        target = f"{target}?{query_string}"
-    return target
-
-
-def _serve_responses_path(request, responses_root: Path, rel_path: str, show_indexes: bool):
-    candidates: list[str] = []
-    rel_path = rel_path or ""
-    if rel_path.endswith("/"):
-        rel_path = f"{rel_path}index.html"
-    if "." not in Path(rel_path).name:
-        candidates.append(f"{rel_path.rstrip('/')}/index.html")
-    candidates.append(rel_path)
-
-    for candidate in candidates:
-        try:
-            return serve_static_with_byterange_support(
-                request,
-                candidate,
-                document_root=str(responses_root),
-                show_indexes=show_indexes,
-                is_archive_replay=True,
-            )
-        except Http404:
-            pass
-
-    if rel_path.endswith("index.html"):
-        rel_dir = rel_path[: -len("index.html")]
-        try:
-            return serve_static_with_byterange_support(
-                request,
-                rel_dir,
-                document_root=str(responses_root),
-                show_indexes=True,
-                is_archive_replay=True,
-            )
-        except Http404:
-            return None
-    return None
-
-
-def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""):
-    rel_path = path or ""
-    request_config = get_request_config(
-        request,
-        resolve_plugins=rel_path.startswith("replay/") or rel_path == "replay",
-    )
-    request.archivebox_config = request_config
-    request.archivebox_snapshot_url = snapshot.url
-    snapshot._runtime_config = request_config
-
-    if rel_path.startswith("replay/") or rel_path == "replay":
-        response = archivewebpage_replay.serve_replay_asset_response(rel_path, request_config, HttpResponse)
-        if response is not None:
-            return response
-
-    if rel_path == "progress.json":
-        # Host routing forwards every snap-* path to SnapshotHostView, so we forward
-        # /progress.json on through to the same view used everywhere else. The caller
-        # passes snapshot_id explicitly in the query string — we don't read it from the
-        # subdomain (this keeps the endpoint identical across all security modes).
-        return live_progress_view(request)
-
-    is_directory_request = bool(path) and path.endswith("/")
-    show_indexes = bool(request.GET.get("files")) or (request_config.USES_SUBDOMAIN_ROUTING and is_directory_request)
-    if not show_indexes and (not rel_path or rel_path == "index.html"):
-        return SnapshotView.render_live_index(request, snapshot)
-
-    if not rel_path or rel_path.endswith("/"):
-        if show_indexes:
-            rel_path = rel_path.rstrip("/")
-        else:
-            rel_path = f"{rel_path}index.html"
-    rel_path = _safe_archive_relpath(rel_path)
-    if rel_path is None:
-        raise Http404
-
-    rel_path, archive_result = _resolve_archiveresult_relpath(snapshot, rel_path)
-
-    plugin_preview = _plugin_full_preview_response(request, snapshot, rel_path, archive_result)
-    if plugin_preview is not None:
-        return plugin_preview
-
-    try:
-        return serve_static_with_byterange_support(
-            request,
-            rel_path,
-            document_root=snapshot.output_dir,
-            show_indexes=show_indexes,
-            is_archive_replay=True,
-        )
-    except Http404:
-        pass
-
-    host = urlparse(snapshot.url).hostname or snapshot.domain
-    responses_root = Path(snapshot.output_dir) / "responses" / host
-    if responses_root.exists():
-        response = _serve_responses_path(request, responses_root, rel_path, show_indexes)
-        if response is not None:
-            return response
-
-    raise Http404
-
-
-def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str = ""):
-    request_config = get_request_config(request, resolve_plugins=False)
-    request.archivebox_config = request_config
-    requested_root_index = path in ("", "index.html") or path.endswith("/")
-    rel_path = path or ""
-    if not rel_path or rel_path.endswith("/"):
-        rel_path = f"{rel_path}index.html"
-    rel_path = _safe_archive_relpath(rel_path)
-    if rel_path is None:
-        raise Http404
-
-    domain = domain.split(":", 1)[0].lower()
-    snapshots = _visible_response_snapshots_for_domain(request, domain)
-    match = _latest_response_match(snapshots, domain, rel_path)
-    if not match and "." not in Path(rel_path).name:
-        index_path = f"{rel_path.rstrip('/')}/index.html"
-        match = _latest_response_match(snapshots, domain, index_path)
-    if not match and "." not in Path(rel_path).name:
-        html_path = f"{rel_path}.html"
-        match = _latest_response_match(snapshots, domain, html_path)
-
-    root_match = (match[0], match[1]) if match else _latest_responses_root(snapshots, domain)
-    responses_root = root_match[1] if root_match else None
-    if request_config.USES_SUBDOMAIN_ROUTING:
-        snapshot = root_match[0] if root_match else (snapshots[0] if requested_root_index and snapshots else None)
-        if snapshot:
-            snapshot_path = f"responses/{domain}/{match[2]}" if match else path
-            target = build_snapshot_url(str(snapshot.id), snapshot_path, request=request, config=request_config)
-            if request.META.get("QUERY_STRING"):
-                target = f"{target}?{request.META['QUERY_STRING']}"
-            return redirect(target)
-
-    show_indexes = bool(request.GET.get("files"))
-    if match:
-        _snapshot, responses_root, rel_to_root = match
-        response = _serve_responses_path(request, responses_root, str(rel_to_root), show_indexes)
-        if response is not None:
-            return response
-
-    if responses_root:
-        response = _serve_responses_path(request, responses_root, rel_path, show_indexes)
-        if response is not None:
-            return response
-
-    if requested_root_index and not show_indexes:
-        if snapshots:
-            return SnapshotView.render_live_index(request, snapshots[0])
-
-    if request_config.PUBLIC_ADD_VIEW or request.user.is_authenticated:
-        target_url = _original_request_url(domain, path, request.META.get("QUERY_STRING", ""))
-        return redirect(build_web_url(f"/web/{quote(target_url, safe=':/')}"))
-
-    raise Http404
-
-
-class SnapshotHostView(View):
-    """Serve snapshot directory contents on <snapshot-subdomain>.<listen_host>/<path>."""
-
-    def get(self, request, snapshot_id: str, path: str = ""):
-        request_config = get_request_config(request)
-        snapshot = _find_snapshot_by_ref(snapshot_id)
-
-        if not snapshot:
-            raise Http404
-        if path == "_auth":
-            return _replay_auth_response(request, snapshot)
-        if not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot):
-            return _private_snapshot_auth_redirect(request, snapshot, path)
-
-        canonical_host = get_snapshot_host(str(snapshot.id), config=request_config)
-        if not host_matches(request.get_host(), canonical_host):
-            target = build_snapshot_url(str(snapshot.id), path, request=request, config=request_config)
-            if request.META.get("QUERY_STRING"):
-                target = f"{target}?{request.META['QUERY_STRING']}"
-            return redirect(target)
-
-        return _serve_snapshot_replay(request, snapshot, path)
-
-
-class SnapshotReplayView(View):
-    """Serve snapshot directory contents on a one-domain replay path."""
-
-    def get(self, request, snapshot_id: str, path: str = ""):
-        snapshot = _find_snapshot_by_ref(snapshot_id)
-        if not snapshot:
-            raise Http404
-        if path == "_auth":
-            return _replay_auth_response(request, snapshot)
-        if not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot):
-            return _private_snapshot_auth_redirect(request, snapshot, path)
-
-        return _serve_snapshot_replay(request, snapshot, path)
-
-
-class OriginalDomainHostView(View):
-    """Serve responses from the most recent snapshot when using <domain>.<listen_host>/<path>."""
-
-    def get(self, request, domain: str, path: str = ""):
-        return _serve_original_domain_replay(request, domain, path)
-
-
-class OriginalDomainReplayView(View):
-    """Serve original-domain replay content on a one-domain replay path."""
-
-    def get(self, request, domain: str, path: str = ""):
-        return _serve_original_domain_replay(request, domain, path)

archivebox/core/views/replay_auth.py

@@ -1,148 +0,0 @@
-from urllib.parse import quote
-
-from django.conf import settings
-from django.contrib.auth import HASH_SESSION_KEY, SESSION_KEY, get_user_model
-from django.contrib.sessions.models import Session
-from django.core import signing
-from django.http import Http404, HttpRequest, HttpResponseForbidden
-from django.shortcuts import redirect
-from django.views import View
-from django.utils.http import url_has_allowed_host_and_scheme
-
-from archivebox.config import CONSTANTS
-from archivebox.config.common import (
-    get_request_config,
-)
-from archivebox.core.models import Snapshot
-from archivebox.core.permissions import (
-    is_admin_user,
-)
-from archivebox.core.routes_util import (
-    build_admin_url,
-    build_snapshot_url,
-)
-
-from .lookup import _find_snapshot_by_ref
-
-REPLAY_AUTH_SALT = "archivebox.private-snapshot-replay"
-REPLAY_COOKIE_PREFIX = f"archivebox_replay_{CONSTANTS.COLLECTION_ID}_"
-REPLAY_GRANT_MAX_AGE = 60
-
-
-def _admin_login_redirect_or_forbidden(request: HttpRequest):
-    if get_request_config(request).CONTROL_PLANE_ENABLED:
-        return redirect(f"/admin/login/?next={request.path}")
-    return HttpResponseForbidden("ArchiveBox is running with the control plane disabled in this security mode.")
-
-
-def _replay_cookie_name(snapshot: Snapshot) -> str:
-    return f"{REPLAY_COOKIE_PREFIX}{str(snapshot.id).replace('-', '')[-12:]}"
-
-
-def _clean_replay_next(path: str | None) -> str:
-    """Only allow same-snap relative replay paths; grants must never redirect off-host."""
-    path = f"/{(path or 'index.html').lstrip('/')}"
-    if not url_has_allowed_host_and_scheme(path, allowed_hosts=set()):
-        return "/index.html"
-    return path
-
-
-def _replay_payload_is_valid(payload: dict, snapshot: Snapshot) -> bool:
-    """A replay cookie is not its own auth source; it must point at a live admin session.
-
-    Replayed pages can execute hostile JS, so admin cookies stay host-only on admin.*.
-    The snap host gets only this host-only HttpOnly cookie, and every request checks
-    that the original Django session still exists and still belongs to an active staff user.
-    Logout, session expiry, user deletion/deactivation, or password auth-hash rotation all
-    make the replay cookie inert without needing admin.* to delete a cookie on snap-*.
-    """
-    if payload.get("snapshot_id") != str(snapshot.id):
-        return False
-    try:
-        session = Session.objects.get(session_key=str(payload.get("session_key") or ""))
-        session_data = session.get_decoded()
-        user_id = str(session_data.get(SESSION_KEY) or "")
-        auth_hash = str(session_data.get(HASH_SESSION_KEY) or "")
-        user = get_user_model().objects.get(pk=user_id)
-    except (Session.DoesNotExist, get_user_model().DoesNotExist, KeyError, TypeError, ValueError):
-        return False
-    return (
-        str(payload.get("user_id")) == user_id
-        and str(payload.get("auth_hash") or "") == auth_hash
-        and user.is_active
-        and user.is_staff
-        and auth_hash == user.get_session_auth_hash()
-    )
-
-
-def _has_replay_cookie(request: HttpRequest, snapshot: Snapshot) -> bool:
-    value = request.COOKIES.get(_replay_cookie_name(snapshot))
-    if not value:
-        return False
-    try:
-        payload = signing.loads(value, salt=REPLAY_AUTH_SALT, max_age=settings.SESSION_COOKIE_AGE)
-    except signing.BadSignature:
-        return False
-    return isinstance(payload, dict) and _replay_payload_is_valid(payload, snapshot)
-
-
-def _private_snapshot_auth_redirect(request: HttpRequest, snapshot: Snapshot, path: str = "", *, preserve_query: bool = True):
-    next_path = _clean_replay_next(path or "index.html")
-    if preserve_query and request.META.get("QUERY_STRING"):
-        next_path = f"{next_path}?{request.META['QUERY_STRING']}"
-    target = build_admin_url(
-        f"/admin/core/snapshot/replay-auth/?snapshot={snapshot.id}&next={quote(next_path, safe='')}",
-        request=request,
-    )
-    return redirect(target)
-
-
-def _replay_auth_response(request: HttpRequest, snapshot: Snapshot):
-    try:
-        payload = signing.loads(str(request.GET.get("grant") or ""), salt=REPLAY_AUTH_SALT, max_age=REPLAY_GRANT_MAX_AGE)
-    except signing.BadSignature:
-        return _private_snapshot_auth_redirect(request, snapshot, "index.html", preserve_query=False)
-
-    if not isinstance(payload, dict) or not _replay_payload_is_valid(payload, snapshot):
-        return _private_snapshot_auth_redirect(request, snapshot, "index.html", preserve_query=False)
-
-    cookie_value = signing.dumps(payload, salt=REPLAY_AUTH_SALT)
-    response = redirect(_clean_replay_next(request.GET.get("next")))
-    response.set_cookie(
-        _replay_cookie_name(snapshot),
-        cookie_value,
-        max_age=settings.SESSION_COOKIE_AGE,
-        secure=request.is_secure(),
-        httponly=True,
-        samesite="Lax",
-    )
-    return response
-
-
-class SnapshotReplayAuthView(View):
-    """Admin-only handoff that lets a snap host mint its own replay cookie.
-
-    admin.* cannot set a host-only cookie for snap-* (browsers forbid that), and
-    widening the real Django session cookie to *.archivebox.localhost would let XSS
-    in replayed pages hit the admin UI. Instead admin.* proves the user is logged in
-    with a short URL grant, then snap-* validates it and sets a snap-host-only cookie.
-    """
-
-    def get(self, request: HttpRequest):
-        if not is_admin_user(request):
-            return redirect(f"{build_admin_url('/admin/login/', request=request)}?next={quote(request.get_full_path(), safe='')}")
-
-        snapshot = _find_snapshot_by_ref(str(request.GET.get("snapshot") or ""))
-        if not snapshot:
-            raise Http404
-
-        payload = {
-            "snapshot_id": str(snapshot.id),
-            "user_id": str(request.user.pk),
-            "session_key": request.session.session_key,
-            "auth_hash": request.user.get_session_auth_hash(),
-        }
-        grant = signing.dumps(payload, salt=REPLAY_AUTH_SALT)
-        next_path = _clean_replay_next(request.GET.get("next"))
-        target = build_snapshot_url(str(snapshot.id), "_auth", request=request, config=get_request_config(request))
-        return redirect(f"{target}?grant={quote(grant, safe='')}&next={quote(next_path, safe='')}")

archivebox/core/widgets.py

@@ -6,7 +6,6 @@ import hashlib
 from django import forms
 from django.db.models.manager import BaseManager
 from django.db.models.query import QuerySet
-from django.template.loader import render_to_string
 from django.utils.html import escape
 from django.utils.safestring import mark_safe
 
@@ -20,7 +19,11 @@ class TagEditorWidget(forms.Widget):
     - Uses AJAX for autocomplete and tag creation
     """
 
-    template_name = "admin/widgets/tags.html"
+    template_name = ""  # We render manually
+
+    class Media:
+        css = {"all": []}
+        js = []
 
     def __init__(self, attrs=None, snapshot_id=None):
         self.snapshot_id = snapshot_id
@@ -37,6 +40,16 @@ class TagEditorWidget(forms.Widget):
             normalized = f"t_{normalized}"
         return normalized
 
+    def _json_for_inline_script(self, value):
+        """Serialize JSON so it cannot close the surrounding inline <script> tag."""
+        return json.dumps(value).translate(
+            {
+                ord(">"): "\\u003E",
+                ord("<"): "\\u003C",
+                ord("&"): "\\u0026",
+            },
+        )
+
     def _tag_style(self, value):
         """Compute a stable pastel color style for a tag value."""
         tag = (value or "").strip().lower()
@@ -93,10 +106,293 @@ class TagEditorWidget(forms.Widget):
         widget_id_raw = attrs.get("id", name) if attrs else name
         widget_id = self._normalize_id(widget_id_raw)
 
-        return render_to_string(
-            self.template_name,
-            {"name": name, "widget_id": widget_id, "tags_id": f"{widget_id}_tags", "tags": tags},
-        )
+        # Build pills HTML
+        pills_html = ""
+        for tag in tags:
+            pills_html += f'''
+                <span class="tag-pill" data-tag="{self._escape(tag)}" style="{self._tag_style(tag)}">
+                    {self._escape(tag)}
+                    <button type="button" class="tag-remove-btn" data-tag-name="{self._escape(tag)}">&times;</button>
+                </span>
+            '''
+
+        tags_json = self._json_for_inline_script(tags)
+
+        # Build the widget HTML
+        html = f'''
+        <div id="{widget_id}_container" class="tag-editor-container" onclick="focusTagInput_{widget_id}(event)">
+            <div id="{widget_id}_pills" class="tag-pills">
+                {pills_html}
+            </div>
+            <input type="text"
+                   id="{widget_id}_input"
+                   class="tag-inline-input"
+                   list="{widget_id}_datalist"
+                   placeholder="Add tag..."
+                   autocomplete="off"
+                   onkeydown="handleTagKeydown_{widget_id}(event)"
+                   onkeypress="if(event.key==='Enter' || event.keyCode===13 || event.key===' ' || event.code==='Space' || event.key==='Spacebar'){{event.preventDefault(); event.stopPropagation();}}"
+                   oninput="fetchTagAutocomplete_{widget_id}(this.value)"
+            >
+            <datalist id="{widget_id}_datalist"></datalist>
+            <input type="hidden" name="{name}" id="{widget_id}" value="{self._escape(",".join(tags))}">
+        </div>
+
+        <script>
+        (function() {{
+            var currentTags_{widget_id} = {tags_json};
+            var autocompleteTimeout_{widget_id} = null;
+
+            window.focusTagInput_{widget_id} = function(event) {{
+                if (event.target.classList.contains('tag-remove-btn')) return;
+                document.getElementById('{widget_id}_input').focus();
+            }};
+
+            window.updateHiddenInput_{widget_id} = function() {{
+                var hiddenInput = document.getElementById('{widget_id}');
+                if (!hiddenInput) {{
+                    return;
+                }}
+                hiddenInput.value = currentTags_{widget_id}.join(',');
+                hiddenInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
+                hiddenInput.dispatchEvent(new Event('change', {{ bubbles: true }}));
+            }};
+
+            function normalizeTags_{widget_id}(value) {{
+                var rawTags = Array.isArray(value) ? value : String(value || '').split(',');
+                var seen = {{}};
+                return rawTags
+                    .map(function(tag) {{ return String(tag || '').trim(); }})
+                    .filter(function(tag) {{
+                        if (!tag) return false;
+                        var normalized = tag.toLowerCase();
+                        if (seen[normalized]) return false;
+                        seen[normalized] = true;
+                        return true;
+                    }})
+                    .sort(function(a, b) {{
+                        return a.toLowerCase().localeCompare(b.toLowerCase());
+                    }});
+            }}
+
+            window.setTags_{widget_id} = function(value, options) {{
+                currentTags_{widget_id} = normalizeTags_{widget_id}(value);
+                rebuildPills_{widget_id}();
+                if (!(options && options.skipHiddenUpdate)) {{
+                    updateHiddenInput_{widget_id}();
+                }}
+            }};
+
+            window.syncTagEditorFromHidden_{widget_id} = function() {{
+                var hiddenInput = document.getElementById('{widget_id}');
+                if (!hiddenInput) {{
+                    return;
+                }}
+                setTags_{widget_id}(hiddenInput.value, {{ skipHiddenUpdate: true }});
+            }};
+
+            function computeTagStyle_{widget_id}(tagName) {{
+                var hash = 0;
+                var name = String(tagName || '').toLowerCase();
+                for (var i = 0; i < name.length; i++) {{
+                    hash = (hash * 31 + name.charCodeAt(i)) % 360;
+                }}
+                var bg = 'hsl(' + hash + ', 70%, 92%)';
+                var border = 'hsl(' + hash + ', 60%, 82%)';
+                var fg = 'hsl(' + hash + ', 35%, 28%)';
+                return {{ bg: bg, border: border, fg: fg }};
+            }}
+
+            function applyTagStyle_{widget_id}(el, tagName) {{
+                var colors = computeTagStyle_{widget_id}(tagName);
+                el.style.setProperty('--tag-bg', colors.bg);
+                el.style.setProperty('--tag-border', colors.border);
+                el.style.setProperty('--tag-fg', colors.fg);
+            }}
+
+            function getApiKey() {{
+                return (window.ARCHIVEBOX_API_KEY || '').trim();
+            }}
+
+            function buildApiUrl(path) {{
+                var apiKey = getApiKey();
+                if (!apiKey) return path;
+                var sep = path.indexOf('?') !== -1 ? '&' : '?';
+                return path + sep + 'api_key=' + encodeURIComponent(apiKey);
+            }}
+
+            function buildApiHeaders() {{
+                var headers = {{
+                    'Content-Type': 'application/json',
+                }};
+                var apiKey = getApiKey();
+                if (apiKey) headers['X-ArchiveBox-API-Key'] = apiKey;
+                var csrfToken = getCSRFToken();
+                if (csrfToken) headers['X-CSRFToken'] = csrfToken;
+                return headers;
+            }}
+
+            window.addTag_{widget_id} = function(tagName) {{
+                tagName = tagName.trim();
+                if (!tagName) return;
+
+                // Check if tag already exists (case-insensitive)
+                var exists = currentTags_{widget_id}.some(function(t) {{
+                    return t.toLowerCase() === tagName.toLowerCase();
+                }});
+                if (exists) {{
+                    document.getElementById('{widget_id}_input').value = '';
+                    return;
+                }}
+
+                // Add to current tags
+                currentTags_{widget_id}.push(tagName);
+                currentTags_{widget_id} = normalizeTags_{widget_id}(currentTags_{widget_id});
+
+                // Rebuild pills
+                rebuildPills_{widget_id}();
+                updateHiddenInput_{widget_id}();
+
+                // Clear input
+                document.getElementById('{widget_id}_input').value = '';
+
+                // Create tag via API if it doesn't exist (fire and forget)
+                fetch(buildApiUrl('/api/v1/core/tags/create/'), {{
+                    method: 'POST',
+                    headers: buildApiHeaders(),
+                    body: JSON.stringify({{ name: tagName }})
+                }}).catch(function(err) {{
+                    console.log('Tag creation note:', err);
+                }});
+            }};
+
+            window.removeTag_{widget_id} = function(tagName) {{
+                currentTags_{widget_id} = currentTags_{widget_id}.filter(function(t) {{
+                    return t.toLowerCase() !== tagName.toLowerCase();
+                }});
+                rebuildPills_{widget_id}();
+                updateHiddenInput_{widget_id}();
+            }};
+
+            window.rebuildPills_{widget_id} = function() {{
+                var container = document.getElementById('{widget_id}_pills');
+                container.innerHTML = '';
+                currentTags_{widget_id}.forEach(function(tag) {{
+                    var pill = document.createElement('span');
+                    pill.className = 'tag-pill';
+                    pill.setAttribute('data-tag', tag);
+                    applyTagStyle_{widget_id}(pill, tag);
+
+                    var tagText = document.createTextNode(tag);
+                    pill.appendChild(tagText);
+
+                    var removeBtn = document.createElement('button');
+                    removeBtn.type = 'button';
+                    removeBtn.className = 'tag-remove-btn';
+                    removeBtn.setAttribute('data-tag-name', tag);
+                    removeBtn.innerHTML = '&times;';
+                    pill.appendChild(removeBtn);
+
+                    container.appendChild(pill);
+                }});
+            }};
+
+            // Add event delegation for remove buttons
+            document.getElementById('{widget_id}_pills').addEventListener('click', function(event) {{
+                if (event.target.classList.contains('tag-remove-btn')) {{
+                    var tagName = event.target.getAttribute('data-tag-name');
+                    if (tagName) {{
+                        removeTag_{widget_id}(tagName);
+                    }}
+                }}
+            }});
+
+            document.getElementById('{widget_id}').addEventListener('change', function() {{
+                syncTagEditorFromHidden_{widget_id}();
+            }});
+
+            document.getElementById('{widget_id}').addEventListener('archivebox:sync-tags', function() {{
+                syncTagEditorFromHidden_{widget_id}();
+            }});
+
+            window.handleTagKeydown_{widget_id} = function(event) {{
+                var input = event.target;
+                var value = input.value.trim();
+                var isSpace = event.key === ' ' || event.code === 'Space' || event.key === 'Spacebar';
+                var isEnter = event.key === 'Enter' || event.keyCode === 13;
+                var isComma = event.key === ',';
+
+                if (isEnter || isSpace || isComma) {{
+                    event.preventDefault();
+                    event.stopPropagation();
+                    if (value) {{
+                        // Treat commas and whitespace as tag boundaries.
+                        value.split(/[\\s,]+/).forEach(function(tag) {{
+                            addTag_{widget_id}(tag.trim());
+                        }});
+                    }}
+                    return false;
+                }} else if (event.key === 'Backspace' && !value && currentTags_{widget_id}.length > 0) {{
+                    // Remove last tag on backspace when input is empty
+                    var lastTag = currentTags_{widget_id}.pop();
+                    rebuildPills_{widget_id}();
+                    updateHiddenInput_{widget_id}();
+                }}
+            }};
+
+            window.fetchTagAutocomplete_{widget_id} = function(query) {{
+                if (autocompleteTimeout_{widget_id}) {{
+                    clearTimeout(autocompleteTimeout_{widget_id});
+                }}
+
+                autocompleteTimeout_{widget_id} = setTimeout(function() {{
+                    if (!query || query.length < 1) {{
+                        document.getElementById('{widget_id}_datalist').innerHTML = '';
+                        return;
+                    }}
+
+                    fetch(buildApiUrl('/api/v1/core/tags/autocomplete/?q=' + encodeURIComponent(query)))
+                        .then(function(response) {{ return response.json(); }})
+                        .then(function(data) {{
+                            var datalist = document.getElementById('{widget_id}_datalist');
+                            datalist.innerHTML = '';
+                            (data.tags || []).forEach(function(tag) {{
+                                var option = document.createElement('option');
+                                option.value = tag.name;
+                                datalist.appendChild(option);
+                            }});
+                        }})
+                        .catch(function(err) {{
+                            console.log('Autocomplete error:', err);
+                        }});
+                }}, 150);
+            }};
+
+            function escapeHtml(text) {{
+                var div = document.createElement('div');
+                div.textContent = text;
+                return div.innerHTML;
+            }}
+
+            function getCSRFToken() {{
+                var cookies = document.cookie.split(';');
+                for (var i = 0; i < cookies.length; i++) {{
+                    var cookie = cookies[i].trim();
+                    if (cookie.startsWith('csrftoken=')) {{
+                        return cookie.substring('csrftoken='.length);
+                    }}
+                }}
+                // Fallback to hidden input
+                var input = document.querySelector('input[name="csrfmiddlewaretoken"]');
+                return input ? input.value : '';
+            }}
+
+            syncTagEditorFromHidden_{widget_id}();
+        }})();
+        </script>
+        '''
+
+        return mark_safe(html)
 
 
 class URLFiltersWidget(forms.Widget):
@@ -452,58 +748,3 @@ class InlineTagEditorWidget(TagEditorWidget):
         '''
 
         return mark_safe(html)
-
-
-def render_copy_block(text: str, *, multiline: bool = False, copy_text: str | None = None):
-    return render_to_string(
-        "admin/widgets/copy_block.html",
-        {
-            "text": text,
-            "copy_text": text if copy_text is None else copy_text,
-            "element": "pre" if multiline else "code",
-            "multiline": multiline,
-        },
-    )
-
-
-def render_permissions_badge(permissions, *, url: str, object_name: str):
-    from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_META, normalize_permissions
-
-    permissions = normalize_permissions(permissions)
-    icon, label, fg, bg = PERMISSIONS_META[permissions]
-    return render_to_string(
-        "admin/widgets/permissions_badge.html",
-        {
-            "permissions": permissions,
-            "url": url,
-            "object_name": object_name,
-            "icon": icon,
-            "label": label,
-            "fg": fg,
-            "bg": bg,
-            "choices": [(value, *PERMISSIONS_META[value]) for value, _label in PERMISSIONS_CHOICES],
-        },
-    )
-
-
-def render_snapshot_progress(stats: dict, *, successful_plugins=None):
-    """Render the same hook counts and progress bar on snapshot cards and rows."""
-    from archivebox.plugins.discovery import get_plugin_icon
-
-    counts = {name: int(stats.get(name) or 0) for name in ("total", "succeeded", "failed", "skipped", "noresults", "running")}
-    total = counts["total"]
-    completed = sum(counts[name] for name in ("succeeded", "failed", "skipped", "noresults"))
-    plugins = sorted(successful_plugins or ())
-    return render_to_string(
-        "admin/widgets/snapshot_progress.html",
-        {
-            **counts,
-            "completed": completed,
-            "percent": int(completed / total * 100) if total else 0,
-            "succeeded_percent": int(counts["succeeded"] / total * 100) if total else 0,
-            "failed_percent": int((counts["succeeded"] + counts["failed"]) / total * 100) if total else 0,
-            "show_plugins": successful_plugins is not None,
-            "plugins": [(name, mark_safe(icon)) for name in plugins[:8] if str(icon := get_plugin_icon(name)).strip()],
-            "remaining_plugins": max(len(plugins) - 8, 0),
-        },
-    )

archivebox/crawls/admin.py

@@ -1,28 +1,40 @@
 __package__ = "archivebox.crawls"
 
+import json
+from copy import copy
 from typing import ClassVar
+from urllib.parse import urlencode, urlparse
 
+from django import forms
 from django.contrib import admin, messages
-from django.db.models import Count, Q
+from django.core.paginator import Paginator
+from django.db.models import Count, F, Prefetch, Q
 from django.http import HttpRequest, HttpResponseBadRequest, HttpResponseNotAllowed, JsonResponse
 from django.shortcuts import get_object_or_404, redirect
+from django.template.loader import render_to_string
 from django.template.response import TemplateResponse
 from django.urls import path, reverse
 from django.utils import timezone
-from django.utils.html import format_html, format_html_join
+from django.utils.html import escape, format_html, format_html_join
 from django.utils.safestring import mark_safe
 from django_object_actions import action
 
-from archivebox.core.widgets import render_permissions_badge
-from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
-from archivebox.core.models import Snapshot
+from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
+from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.core.permissions import (
+    PERMISSIONS_CHOICES,
     PERMISSIONS_META,
+    PERMISSIONS_PRIVATE,
+    PERMISSIONS_PUBLIC,
+    PERMISSIONS_UNLISTED,
     PERMISSIONS_VALUES,
+    normalize_permissions,
 )
+from archivebox.core.routes_util import build_snapshot_detail_url, build_snapshot_role_output_url
+from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
 from archivebox.crawls.models import Crawl, CrawlSchedule
-from archivebox.crawls.forms import CrawlAdminForm
 from archivebox.misc.paginators import AcceleratedPaginator
+from archivebox.plugins.discovery import get_snapshot_role_names
 from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.workers.models import RETRY_AT_MAX
 
@@ -41,6 +53,581 @@ class MaxDepthListFilter(admin.SimpleListFilter):
         return queryset
 
 
+def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50, prefix="snapshots"):
+    """Render a nice inline list view of snapshots with status, title, URL, and progress."""
+
+    query_param = f"{prefix}_q"
+    status_param = f"{prefix}_status"
+    page_param = f"{prefix}_page"
+    query = (request.GET.get(query_param, "") if request is not None else "").strip()
+    status_filter = (request.GET.get(status_param, "") if request is not None else "").strip()
+    valid_statuses = {choice[0] for choice in Snapshot.StatusChoices.choices}
+
+    filtered_qs = snapshots_qs
+    if query:
+        from archivebox.misc.util import filter_queryset_by_uuid_substring
+
+        id_match_pks = list(filter_queryset_by_uuid_substring(Snapshot.objects.all(), query).values_list("pk", flat=True)[:100])
+        filtered_qs = filtered_qs.filter(Q(pk__in=id_match_pks) | Q(url__icontains=query) | Q(title__icontains=query))
+    if status_filter in valid_statuses:
+        filtered_qs = filtered_qs.filter(status=status_filter)
+
+    # Keep ArchiveResult counters as scalar subqueries so the paginated
+    # Snapshot queryset does not become a join+GROUP BY over every result row.
+    snapshots_qs = (
+        filtered_qs.order_by("-created_at")
+        .annotate(
+            total_results=ArchiveResult.snapshot_count_expr(),
+            succeeded_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
+            failed_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.FAILED),
+            started_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.STARTED),
+            skipped_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SKIPPED),
+            snapshot_permissions=F("permissions"),
+        )
+        .prefetch_related(
+            Prefetch(
+                "archiveresult_set",
+                queryset=ArchiveResult.objects.filter(
+                    plugin__in=get_snapshot_role_names("list_icon"),
+                    status=ArchiveResult.StatusChoices.SUCCEEDED,
+                ).only(
+                    "snapshot_id",
+                    "plugin",
+                    "status",
+                    "output_str",
+                    "output_files",
+                ),
+                to_attr="_list_icon_results",
+            ),
+        )
+    )
+
+    page_number = request.GET.get(page_param, 1) if request is not None else 1
+    paginator = Paginator(snapshots_qs, page_size)
+    page_obj = paginator.get_page(page_number)
+    snapshots = page_obj.object_list
+    total_count = paginator.count
+
+    def querystring(**updates):
+        if request is None:
+            return "#"
+        params = request.GET.copy()
+        for key, value in updates.items():
+            if value in (None, ""):
+                params.pop(key, None)
+            else:
+                params[key] = str(value)
+        return f"?{params.urlencode()}" if params else "?"
+
+    preserved_inputs = ""
+    if request is not None:
+        managed_params = {query_param, status_param, page_param}
+        preserved_inputs = "".join(
+            f'<input type="hidden" name="{escape(key)}" value="{escape(value)}">'
+            for key, values in request.GET.lists()
+            if key not in managed_params
+            for value in values
+        )
+
+    status_options = "".join(
+        f'<option value="{escape(value)}"{" selected" if status_filter == value else ""}>{escape(label)}</option>'
+        for value, label in Snapshot.StatusChoices.choices
+    )
+
+    controls = f"""
+        <div class="crawl-snapshots-toolbar" style="display: flex; gap: 10px; align-items: center; justify-content: space-between; flex-wrap: wrap; padding: 10px 12px; background: #f8fafc; border-bottom: 1px solid #e2e8f0;">
+            <form method="get" style="display: flex; gap: 8px; align-items: center; flex: 1 1 540px; margin: 0;">
+                {preserved_inputs}
+                <input type="search" name="{query_param}" value="{escape(query)}" placeholder="Filter snapshots by title, URL, or ID"
+                       style="min-width: 260px; flex: 1 1 360px; padding: 7px 10px; border: 1px solid #cbd5e1; border-radius: 6px;">
+                <select name="{status_param}" style="max-width: 170px; padding: 7px 10px; border: 1px solid #cbd5e1; border-radius: 6px;">
+                    <option value="">All statuses</option>
+                    {status_options}
+                </select>
+                <input type="hidden" name="{page_param}" value="1">
+                <button type="submit" class="button" style="padding: 7px 12px;">Filter</button>
+                {f'<a href="{querystring(**{query_param: None, status_param: None, page_param: None})}" style="font-size: 12px; color: #64748b;">Clear</a>' if query or status_filter else ""}
+            </form>
+            <div style="font-size: 12px; color: #64748b; white-space: nowrap;">
+                {page_obj.start_index() if total_count else 0}-{page_obj.end_index() if total_count else 0} of {total_count}
+            </div>
+        </div>
+    """
+
+    if not snapshots:
+        return mark_safe(f"""
+            <div data-crawl-snapshots-list style="border: 1px solid #ddd; border-radius: 6px; overflow: hidden; max-width: 100%;">
+                {controls}
+                <div style="color: #666; font-style: italic; padding: 12px;">No Snapshots found.</div>
+            </div>
+        """)
+
+    # Status colors matching Django admin and progress monitor
+    status_colors = {
+        "queued": ("#6c757d", "#f8f9fa"),  # gray
+        "started": ("#856404", "#fff3cd"),  # amber
+        "paused": ("#1d4ed8", "#dbeafe"),  # blue
+        "sealed": ("#155724", "#d4edda"),  # green
+        "failed": ("#721c24", "#f8d7da"),  # red
+    }
+
+    rows = []
+    for snapshot in snapshots:
+        status = snapshot.status or "queued"
+        color, bg = status_colors.get(status, ("#6c757d", "#f8f9fa"))
+        permissions = snapshot.snapshot_permissions
+        permission_icon = {
+            PERMISSIONS_PUBLIC: "👁",
+            PERMISSIONS_UNLISTED: "🔗",
+            PERMISSIONS_PRIVATE: "🔒",
+        }[permissions]
+        permission_fg, permission_bg = {
+            PERMISSIONS_PUBLIC: ("#047857", "#d1fae5"),
+            PERMISSIONS_UNLISTED: ("#1d4ed8", "#dbeafe"),
+            PERMISSIONS_PRIVATE: ("#991b1b", "#fee2e2"),
+        }[permissions]
+
+        # Calculate progress
+        total = snapshot.total_results
+        succeeded = snapshot.succeeded_results
+        failed = snapshot.failed_results
+        running = snapshot.started_results
+        skipped = snapshot.skipped_results
+        done = succeeded + failed + skipped
+        pending = max(total - done - running, 0)
+        progress_pct = int((done / total) * 100) if total > 0 else 0
+        progress_text = f"{done}/{total}" if total > 0 else "-"
+        progress_title = f"{succeeded} succeeded, {failed} failed, {running} running, {pending} pending, {skipped} skipped"
+        progress_color = "#28a745"
+        if failed:
+            progress_color = "#dc3545"
+        elif running:
+            progress_color = "#17a2b8"
+        elif pending:
+            progress_color = "#ffc107"
+
+        # Truncate title and URL
+        snapshot_title = snapshot.title or "Untitled"
+        title = snapshot_title[:60]
+        if len(snapshot_title) > 60:
+            title += "..."
+        url_display = snapshot.url[:50]
+        if len(snapshot.url) > 50:
+            url_display += "..."
+        delete_button = ""
+        exclude_button = ""
+        if crawl is not None:
+            delete_url = reverse("admin:crawls_crawl_snapshot_delete", args=[crawl.pk, snapshot.pk])
+            exclude_url = reverse("admin:crawls_crawl_snapshot_exclude_domain", args=[crawl.pk, snapshot.pk])
+            delete_button = f'''
+                <button type="button"
+                        class="crawl-snapshots-action"
+                        data-post-url="{escape(delete_url)}"
+                        data-confirm="Delete this snapshot from the crawl?"
+                        title="Delete this snapshot from the crawl and remove its URL from the crawl queue."
+                        aria-label="Delete snapshot"
+                        style="border: 1px solid #ddd; background: #fff; color: #666; border-radius: 4px; width: 28px; height: 28px; cursor: pointer;">🗑</button>
+            '''
+            exclude_button = f'''
+                <button type="button"
+                        class="crawl-snapshots-action"
+                        data-post-url="{escape(exclude_url)}"
+                        data-confirm="Exclude this domain from the crawl? This removes matching queued URLs, deletes pending matching snapshots, and blocks future matches."
+                        title="Exclude this domain from this crawl. This removes matching URLs from the crawl queue, deletes pending matching snapshots, and blocks future matches."
+                        aria-label="Exclude domain from crawl"
+                        style="border: 1px solid #ddd; background: #fff; color: #666; border-radius: 4px; width: 28px; height: 28px; cursor: pointer;">⊘</button>
+            '''
+
+        # Format date
+        date_str = snapshot.created_at.strftime("%Y-%m-%d %H:%M") if snapshot.created_at else "-"
+        detail_url = build_snapshot_detail_url(snapshot.archive_path_from_db, request=request)
+        favicon_url = build_snapshot_role_output_url(
+            snapshot,
+            "list_icon",
+            request=request,
+            archive_results=snapshot._list_icon_results,
+            fallback_to_default=True,
+        )
+
+        rows.append(f'''
+            <tr style="border-bottom: 1px solid #eee;">
+                <td style="padding: 6px 8px; white-space: nowrap;">
+                    <span style="display: inline-block; padding: 2px 8px; border-radius: 10px;
+                                 font-size: 11px; font-weight: 500; text-transform: uppercase;
+                                 color: {color}; background: {bg};">{status}</span>
+                </td>
+                <td style="padding: 6px 8px; white-space: nowrap; text-align: center;">
+                    <span title="{permissions}" style="display:inline-flex; align-items:center; justify-content:center; width:22px; height:22px; border-radius:999px; font-size:12px; color:{permission_fg}; background:{permission_bg};">{permission_icon}</span>
+                </td>
+                <td style="padding: 6px 8px; white-space: nowrap;">
+                    <a href="{escape(detail_url)}" style="text-decoration: none;">
+                        <img src="{escape(favicon_url)}"
+                             style="width: 16px; height: 16px; vertical-align: middle; margin-right: 4px;"
+                             onerror="this.style.display='none'"/>
+                    </a>
+                </td>
+                <td style="padding: 6px 8px; max-width: 300px;">
+                    <a href="{snapshot.admin_change_url}" style="color: #417690; text-decoration: none; font-weight: 500;"
+                       title="{escape(snapshot_title)}">{escape(title)}</a>
+                </td>
+                <td style="padding: 6px 8px; max-width: 250px;">
+                    <a href="{escape(snapshot.url)}" target="_blank"
+                       style="color: #666; text-decoration: none; font-family: monospace; font-size: 11px;"
+                       title="{escape(snapshot.url)}">{escape(url_display)}</a>
+                </td>
+                <td style="padding: 6px 8px; white-space: nowrap; text-align: center;">
+                    <div style="display: inline-flex; align-items: center; gap: 6px;" title="{escape(progress_title)}">
+                        <div style="width: 60px; height: 6px; background: #eee; border-radius: 3px; overflow: hidden;">
+                            <div style="width: {progress_pct}%; height: 100%;
+                                        background: {progress_color};
+                                        transition: width 0.3s;"></div>
+                        </div>
+                        <a href="/admin/core/archiveresult/?snapshot__id__exact={snapshot.id}"
+                           style="font-size: 11px; color: #417690; min-width: 35px; text-decoration: none;"
+                           title="View archive results">{progress_text}</a>
+                    </div>
+                </td>
+                <td style="padding: 6px 8px; white-space: nowrap; color: #888; font-size: 11px;">
+                    {date_str}
+                </td>
+                {f'<td style="padding: 6px 8px; white-space: nowrap; text-align: right;"><div style="display: inline-flex; gap: 6px;">{exclude_button}{delete_button}</div></td>' if crawl is not None else ""}
+            </tr>
+        ''')
+
+    pagination = ""
+    if paginator.num_pages > 1:
+        pagination = f"""
+            <div style="display: flex; gap: 10px; align-items: center; justify-content: center; padding: 10px 12px; background: #f8fafc; border-top: 1px solid #e2e8f0; font-size: 12px;">
+                {"<a class='button' style='padding: 5px 10px;' href='" + querystring(**{page_param: page_obj.previous_page_number()}) + "'>Previous</a>" if page_obj.has_previous() else "<span style='color:#94a3b8;'>Previous</span>"}
+                <span style="color: #64748b;">Page {page_obj.number} of {paginator.num_pages}</span>
+                {"<a class='button' style='padding: 5px 10px;' href='" + querystring(**{page_param: page_obj.next_page_number()}) + "'>Next</a>" if page_obj.has_next() else "<span style='color:#94a3b8;'>Next</span>"}
+            </div>
+        """
+
+    return mark_safe(f"""
+        <div data-crawl-snapshots-list style="border: 1px solid #ddd; border-radius: 6px; overflow: hidden; max-width: 100%;">
+            {controls}
+            <table style="width: 100%; border-collapse: collapse; font-size: 13px;">
+                <thead>
+                    <tr style="background: #f5f5f5; border-bottom: 2px solid #ddd;">
+                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">Status</th>
+                        <th style="padding: 8px 4px; text-align: center; font-weight: 600; color: #333; width: 22px;">🔒</th>
+                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333; width: 24px;"></th>
+                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">Title</th>
+                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">URL</th>
+                        <th style="padding: 8px; text-align: center; font-weight: 600; color: #333;">Progress</th>
+                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">Created</th>
+                        {
+        '<th style="padding: 8px; text-align: right; font-weight: 600; color: #333;">Actions</th>' if crawl is not None else ""
+    }
+                    </tr>
+                </thead>
+                <tbody>
+                    {"".join(rows)}
+                </tbody>
+            </table>
+            {pagination}
+        </div>
+        {
+        '''
+        <script>
+        (function() {
+            if (window.__archiveboxCrawlSnapshotActionsBound) {
+                return;
+            }
+            window.__archiveboxCrawlSnapshotActionsBound = true;
+
+            function getCookie(name) {
+                var cookieValue = null;
+                if (!document.cookie) {
+                    return cookieValue;
+                }
+                var cookies = document.cookie.split(';');
+                for (var i = 0; i < cookies.length; i++) {
+                    var cookie = cookies[i].trim();
+                    if (cookie.substring(0, name.length + 1) === (name + '=')) {
+                        cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
+                        break;
+                    }
+                }
+                return cookieValue;
+            }
+
+            document.addEventListener('click', function(event) {
+                var button = event.target.closest('.crawl-snapshots-action');
+                if (!button) {
+                    return;
+                }
+                event.preventDefault();
+
+                var confirmMessage = button.getAttribute('data-confirm');
+                if (confirmMessage && !window.confirm(confirmMessage)) {
+                    return;
+                }
+
+                button.disabled = true;
+
+                fetch(button.getAttribute('data-post-url'), {
+                    method: 'POST',
+                    credentials: 'same-origin',
+                    headers: {
+                        'X-CSRFToken': getCookie('csrftoken') || '',
+                        'X-Requested-With': 'XMLHttpRequest'
+                    }
+                }).then(function(response) {
+                    return response.json().then(function(data) {
+                        if (!response.ok) {
+                            throw new Error(data.error || 'Request failed');
+                        }
+                        return data;
+                    });
+                }).then(function() {
+                    window.location.reload();
+                }).catch(function(error) {
+                    button.disabled = false;
+                    window.alert(error.message || 'Request failed');
+                });
+            });
+        })();
+        </script>
+        '''
+        if crawl is not None
+        else ""
+    }
+    """)
+
+
+class URLFiltersField(forms.Field):
+    widget = URLFiltersWidget(source_selector="#id_urls")
+
+    def to_python(self, value):
+        if isinstance(value, dict):
+            return value
+        return {"allowlist": "", "denylist": "", "same_domain_only": False, "subpaths_only": False, "only_new": False}
+
+
+class CrawlAdminForm(forms.ModelForm):
+    """Custom form for Crawl admin to render urls field as textarea."""
+
+    tags_editor = forms.CharField(
+        label="Tags",
+        required=False,
+        widget=TagEditorWidget(),
+        help_text="Type tag names and press Enter or Space to add. Click × to remove.",
+    )
+    url_filters = URLFiltersField(
+        label="URL Filters",
+        required=False,
+        help_text="Set URL_ALLOWLIST / URL_DENYLIST for this crawl.",
+    )
+
+    class Meta:
+        model = Crawl
+        fields = "__all__"
+        widgets: ClassVar[dict[str, forms.Widget]] = {
+            "urls": forms.Textarea(
+                attrs={
+                    "rows": 8,
+                    "style": "width: 100%; font-family: monospace; font-size: 13px;",
+                    "placeholder": "https://example.com\nhttps://example2.com\n# Comments start with #",
+                },
+            ),
+            "notes": forms.Textarea(
+                attrs={
+                    "rows": 1,
+                    "style": "width: 100%; min-height: 0; resize: vertical;",
+                },
+            ),
+        }
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        config = dict(self.instance.config or {}) if self.instance and self.instance.pk else {}
+        if self.instance and self.instance.pk:
+            self.initial["tags_editor"] = self.instance.tags_str
+        effective_only_new = self.effective_only_new(self.instance if self.instance and self.instance.pk else None)
+        derived_filter_toggles = self.derive_filter_toggles(
+            self.instance.urls if self.instance and self.instance.pk else "",
+            config.get("URL_ALLOWLIST", ""),
+        )
+        self.initial["url_filters"] = {
+            "allowlist": config.get("URL_ALLOWLIST", ""),
+            "denylist": config.get("URL_DENYLIST", ""),
+            "same_domain_only": derived_filter_toggles["same_domain_only"],
+            "subpaths_only": derived_filter_toggles["subpaths_only"],
+            "only_new": effective_only_new,
+        }
+
+    @staticmethod
+    def extract_url_line(line):
+        line = str(line or "").strip()
+        if not line or line.startswith("#"):
+            return ""
+        if line.startswith("{"):
+            try:
+                return str(json.loads(line).get("url", "")).strip()
+            except (TypeError, ValueError, json.JSONDecodeError):
+                return ""
+        return line
+
+    @staticmethod
+    def regex_escape(text):
+        escaped = ""
+        for char in str(text or ""):
+            escaped += f"\\{char}" if char in r".*+?^${}()|[]\\" else char
+        return escaped
+
+    @classmethod
+    def generated_host_allowlist(cls, urls):
+        seen = set()
+        domains = []
+        for raw_line in str(urls or "").splitlines():
+            url = cls.extract_url_line(raw_line)
+            if not url:
+                continue
+            parsed = urlparse(url)
+            domain = (parsed.hostname or "").lower()
+            if not domain or domain in seen:
+                continue
+            seen.add(domain)
+            domains.append(domain)
+        if not domains:
+            return ""
+        return "^https?://(" + "|".join(cls.regex_escape(domain) for domain in domains) + ")([:/]|$)"
+
+    @staticmethod
+    def subpath_prefix(pathname):
+        path = str(pathname or "/")
+        while "//" in path:
+            path = path.replace("//", "/")
+        if not path or path == "/":
+            return "/"
+        if path.endswith("/"):
+            return path
+        last_slash = path.rfind("/")
+        last_part = path[last_slash + 1 :]
+        if "." in last_part:
+            return path[: last_slash + 1] or "/"
+        return path
+
+    @staticmethod
+    def parsed_host_and_port(parsed):
+        host = (parsed.hostname or "").lower()
+        if not host:
+            return ""
+        try:
+            port = parsed.port
+        except ValueError:
+            port = None
+        return f"{host}:{port}" if port is not None else host
+
+    @classmethod
+    def generated_subpath_allowlist(cls, urls):
+        seen = set()
+        paths = []
+        for raw_line in str(urls or "").splitlines():
+            url = cls.extract_url_line(raw_line)
+            if not url:
+                continue
+            parsed = urlparse(url)
+            domain = (parsed.hostname or "").lower()
+            if domain:
+                seen.add(domain)
+            host = cls.parsed_host_and_port(parsed)
+            path = cls.subpath_prefix(parsed.path)
+            path_key = f"{host}{path}"
+            if not host or path_key in seen:
+                continue
+            seen.add(path_key)
+            paths.append((host, path))
+        if not paths:
+            return ""
+        patterns = []
+        for host, path in paths:
+            if path == "/":
+                patterns.append(f"^https?://{cls.regex_escape(host)}([/?#]|$)")
+            elif path.endswith("/"):
+                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}")
+            else:
+                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}([/?#]|$)")
+        return "\n".join(patterns)
+
+    @classmethod
+    def derive_filter_toggles(cls, urls, allowlist):
+        normalized_allowlist = "\n".join(Crawl.split_filter_patterns(allowlist))
+        if not normalized_allowlist:
+            return {"same_domain_only": False, "subpaths_only": False}
+        if normalized_allowlist == cls.generated_subpath_allowlist(urls):
+            return {"same_domain_only": True, "subpaths_only": True}
+        if normalized_allowlist == cls.generated_host_allowlist(urls):
+            return {"same_domain_only": True, "subpaths_only": False}
+        return {"same_domain_only": False, "subpaths_only": False}
+
+    @staticmethod
+    def effective_only_new(crawl=None):
+        from archivebox.config.common import get_config
+
+        if crawl is not None:
+            return bool(get_config(crawl=crawl, resolve_plugins=False).ONLY_NEW)
+        return bool(get_config(resolve_plugins=False).ONLY_NEW)
+
+    @staticmethod
+    def inherited_only_new(crawl):
+        crawl_without_only_new = copy(crawl)
+        config = dict(crawl.config or {})
+        config.pop("ONLY_NEW", None)
+        crawl_without_only_new.config = config
+        return CrawlAdminForm.effective_only_new(crawl_without_only_new)
+
+    def clean_tags_editor(self):
+        tags_str = self.cleaned_data.get("tags_editor", "")
+        tag_names = []
+        seen = set()
+        for raw_name in tags_str.split(","):
+            name = raw_name.strip()
+            if not name:
+                continue
+            lowered = name.lower()
+            if lowered in seen:
+                continue
+            seen.add(lowered)
+            tag_names.append(name)
+        return ",".join(tag_names)
+
+    def clean_url_filters(self):
+        value = self.cleaned_data.get("url_filters") or {}
+        return {
+            "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
+            "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
+            "same_domain_only": bool(value.get("same_domain_only")),
+            "subpaths_only": bool(value.get("subpaths_only")),
+            "only_new": bool(value.get("only_new")),
+        }
+
+    def save(self, commit=True):
+        instance = super().save(commit=False)
+        instance.tags_str = self.cleaned_data.get("tags_editor", "")
+        if f"{self.add_prefix('url_filters')}_allowlist" in self.data or f"{self.add_prefix('url_filters')}_denylist" in self.data:
+            url_filters = self.cleaned_data.get("url_filters") or {}
+            instance.set_url_filters(
+                url_filters.get("allowlist", ""),
+                url_filters.get("denylist", ""),
+            )
+            config = dict(instance.config or {})
+            only_new = bool(url_filters.get("only_new"))
+            inherited_only_new = self.inherited_only_new(instance)
+            if only_new != inherited_only_new:
+                config["ONLY_NEW"] = only_new
+            else:
+                config.pop("ONLY_NEW", None)
+            instance.config = config
+        if commit:
+            instance.save()
+            instance.apply_crawl_config_filters()
+            self._save_m2m()
+        return instance
+
+
 class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
     form = CrawlAdminForm
     change_form_template = "admin/crawls/crawl/change_form.html"
@@ -88,7 +675,13 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
     readonly_fields = ("created_at", "modified_at", "stop_reason_display")
 
     fieldsets = (
-        card_fieldset("URLs", ("urls", "url_filters"), wide=True),
+        (
+            "URLs",
+            {
+                "fields": ("urls", "url_filters"),
+                "classes": ("card", "wide"),
+            },
+        ),
         (
             "Overview",
             {
@@ -110,7 +703,13 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         ),
     )
     add_fieldsets = (
-        card_fieldset("URLs", ("urls", "url_filters"), wide=True),
+        (
+            "URLs",
+            {
+                "fields": ("urls", "url_filters"),
+                "classes": ("card", "wide"),
+            },
+        ),
         (
             "Overview",
             {
@@ -454,11 +1053,45 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="👁", ordering="permissions")
     def permissions_badge(self, obj):
-        permissions = obj.permissions
-        return render_permissions_badge(
+        permissions = normalize_permissions(obj.permissions)
+        icon, label, fg, bg = PERMISSIONS_META[permissions]
+        menu_items = format_html_join(
+            "",
+            (
+                '<button type="button" class="snapshot-permissions-menu-item{}" data-permissions="{}">'
+                '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
+                "<span>{}</span>"
+                "</button>"
+            ),
+            (
+                (
+                    " is-active" if choice_value == permissions else "",
+                    choice_value,
+                    choice_fg,
+                    choice_bg,
+                    choice_icon,
+                    choice_label,
+                )
+                for choice_value, choice_label in PERMISSIONS_CHOICES
+                for choice_icon, _choice_title, choice_fg, choice_bg in [PERMISSIONS_META[choice_value]]
+            ),
+        )
+        return format_html(
+            '<span class="snapshot-permissions-quick" data-current-permissions="{}" data-permissions-url="{}">'
+            '<button type="button" class="snapshot-permissions-button snapshot-permissions-{}" title="{}" aria-label="Change crawl permissions: {}" aria-expanded="false">'
+            '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
+            "</button>"
+            '<span class="snapshot-permissions-menu" role="menu" hidden>{}</span>'
+            "</span>",
             permissions,
-            url=reverse(f"{self.admin_site.name}:crawls_crawl_set_permissions", args=[obj.pk]),
-            object_name="crawl",
+            reverse(f"{self.admin_site.name}:crawls_crawl_set_permissions", args=[obj.pk]),
+            permissions,
+            label,
+            label,
+            fg,
+            bg,
+            icon,
+            menu_items,
         )
 
     @admin.display(description="Pause")
@@ -485,11 +1118,36 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Snapshots")
     def snapshots_changelist(self, obj):
-        return self.admin_site._registry[Snapshot].render_embedded_changelist(
-            self.request,
-            filters={"crawl_id": str(obj.pk)},
-            title="Snapshots in this crawl",
+        request = self.request
+        snapshot_changelist = reverse("admin:core_snapshot_changelist")
+        scoped_params = {"crawl_id": str(obj.pk)}
+        full_url = f"{snapshot_changelist}?{urlencode(scoped_params)}"
+
+        snapshot_admin = self.admin_site._registry[Snapshot]
+        changelist_request = copy(request)
+        changelist_request.method = "GET"
+        changelist_request.path = snapshot_changelist
+        changelist_request.GET = request.GET.copy()
+        changelist_request.GET.update(
+            {
+                **scoped_params,
+                "_embedded": "crawl",
+                "per_page": "200",
+            },
         )
+        changelist_request.POST = request.POST.copy()
+        changelist_request.POST.clear()
+
+        response = snapshot_admin.changelist_view(
+            changelist_request,
+            extra_context={"embedded_changelist": True},
+        )
+        context = {
+            **response.context_data,
+            "snapshot_changelist_url": full_url,
+            "crawl": obj,
+        }
+        return mark_safe(render_to_string("admin/crawls/crawl/snapshots_changelist.html", context, request=request))
 
     def delete_snapshot_view(self, request: HttpRequest, object_id: str, snapshot_id: str):
         if request.method != "POST":
@@ -549,25 +1207,83 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         first_url = next((line.strip() for line in (obj.urls or "").splitlines() if line.strip() and not line.strip().startswith("#")), "")
         return first_url[:80] + "..." if len(first_url) > 80 else first_url
 
+    @admin.display(description="URLs")
+    def urls_editor(self, obj):
+        """Editor for crawl URLs."""
+        widget_id = f"crawl_urls_{obj.pk}"
+
+        # Escape for safe HTML embedding
+        escaped_urls = (obj.urls or "").replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
+
+        # Count lines for auto-expand logic
+        line_count = len((obj.urls or "").split("\n"))
+        uri_rows = min(max(3, line_count), 10)
+
+        html = f'''
+        <div id="{widget_id}_container" style="max-width: 900px;">
+            <!-- URLs input -->
+            <div style="margin-bottom: 12px;">
+                <label style="font-weight: bold; display: block; margin-bottom: 4px;">URLs (one per line):</label>
+                <textarea id="{widget_id}_urls"
+                          style="width: 100%; font-family: monospace; font-size: 13px;
+                                 padding: 8px; border: 1px solid #ccc; border-radius: 4px;
+                                 resize: vertical;"
+                          rows="{uri_rows}"
+                          placeholder="https://example.com&#10;https://example2.com&#10;# Comments start with #"
+                          readonly>{escaped_urls}</textarea>
+                <p style="color: #666; font-size: 12px; margin: 4px 0 0 0;">
+                    {line_count} URL{"s" if line_count != 1 else ""} · Note: URLs displayed here for reference only
+                </p>
+            </div>
+        </div>
+        '''
+        return mark_safe(html)
+
 
 class CrawlScheduleAdmin(BaseModelAdmin):
-    change_form_template = "admin/crawls/crawlschedule/change_form.html"
-
-    class Media:
-        css: ClassVar[dict[str, tuple[str, ...]]] = {"all": ("admin/crawls/crawl_change.css",)}
-
     list_display = ("id", "created_at", "created_by", "label", "notes", "template_str", "crawls", "num_crawls", "num_snapshots")
     sort_fields = ("id", "created_at", "created_by", "label", "notes", "template_str")
     search_fields = ("id", "created_by__username", "label", "notes", "schedule_id", "template_id", "template__urls")
 
-    readonly_fields = ("created_at", "modified_at", "crawls")
+    readonly_fields = ("created_at", "modified_at", "crawls", "snapshots")
     autocomplete_fields = ("template", "created_by")
 
     fieldsets = (
-        card_fieldset("Schedule Info", ("label", "notes")),
-        card_fieldset("Configuration", ("is_enabled", "schedule", "template")),
-        card_fieldset("Metadata", ("created_by", "created_at", "modified_at")),
-        card_fieldset("Crawls", ("crawls",), wide=True),
+        (
+            "Schedule Info",
+            {
+                "fields": ("label", "notes"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Configuration",
+            {
+                "fields": ("is_enabled", "schedule", "template"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Metadata",
+            {
+                "fields": ("created_by", "created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Crawls",
+            {
+                "fields": ("crawls",),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Snapshots",
+            {
+                "fields": ("snapshots",),
+                "classes": ("card", "wide"),
+            },
+        ),
     )
 
     list_filter = ("created_by",)
@@ -589,15 +1305,6 @@ class CrawlScheduleAdmin(BaseModelAdmin):
 
     def change_view(self, request, object_id, form_url="", extra_context=None):
         self.request = request
-        schedule = self.get_object(request, object_id)
-        extra_context = {**(extra_context or {})}
-        if schedule:
-            extra_context["schedule_snapshots_changelist"] = self.admin_site._registry[Snapshot].render_embedded_changelist(
-                request,
-                filters={"crawl__schedule__id__exact": str(schedule.pk)},
-                title="Snapshots in this schedule",
-                default_search_mode="meta",
-            )
         from django.db import connections, router
 
         connection = connections[router.db_for_write(self.model)]
@@ -619,7 +1326,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
 
     def get_fieldsets(self, request, obj=None):
         if obj is None:
-            return tuple(fieldset for fieldset in self.fieldsets if fieldset[0] != "Crawls")
+            return tuple(fieldset for fieldset in self.fieldsets if fieldset[0] not in {"Crawls", "Snapshots"})
         return self.fieldsets
 
     def save_model(self, request, obj, form, change):
@@ -652,6 +1359,14 @@ class CrawlScheduleAdmin(BaseModelAdmin):
             ((crawl.admin_change_url, crawl) for crawl in obj.crawl_set.all().order_by("-created_at")[:20]),
         ) or mark_safe("<i>No Crawls yet...</i>")
 
+    def snapshots(self, obj):
+        crawl_ids = obj.crawl_set.values_list("pk", flat=True)
+        return render_snapshots_list(
+            Snapshot.objects.filter(crawl_id__in=crawl_ids),
+            request=self.request,
+            prefix="schedule_snapshots",
+        )
+
 
 def register_admin(admin_site):
     admin_site.register(Crawl, CrawlAdmin)

archivebox/crawls/forms.py

@@ -1,237 +0,0 @@
-"""Crawl form input parsing; lifecycle operations remain on Crawl."""
-
-import json
-from copy import copy
-from typing import ClassVar
-from urllib.parse import urlparse
-
-from django import forms
-
-from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
-from archivebox.crawls.models import Crawl
-
-
-class URLFiltersField(forms.Field):
-    widget = URLFiltersWidget(source_selector="#id_urls")
-
-    def to_python(self, value):
-        value = value if isinstance(value, dict) else {}
-        return {
-            "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
-            "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
-            "same_domain_only": bool(value.get("same_domain_only")),
-            "subpaths_only": bool(value.get("subpaths_only")),
-            "only_new": bool(value.get("only_new")),
-        }
-
-
-class CrawlAdminForm(forms.ModelForm):
-    """Custom form for Crawl admin to render urls field as textarea."""
-
-    tags_editor = forms.CharField(
-        label="Tags",
-        required=False,
-        widget=TagEditorWidget(),
-        help_text="Type tag names and press Enter or Space to add. Click × to remove.",
-    )
-    url_filters = URLFiltersField(
-        label="URL Filters",
-        required=False,
-        help_text="Set URL_ALLOWLIST / URL_DENYLIST for this crawl.",
-    )
-
-    class Meta:
-        model = Crawl
-        fields = "__all__"
-        widgets: ClassVar[dict[str, forms.Widget]] = {
-            "urls": forms.Textarea(
-                attrs={
-                    "rows": 8,
-                    "style": "width: 100%; font-family: monospace; font-size: 13px;",
-                    "placeholder": "https://example.com\nhttps://example2.com\n# Comments start with #",
-                },
-            ),
-            "notes": forms.Textarea(
-                attrs={
-                    "rows": 1,
-                    "style": "width: 100%; min-height: 0; resize: vertical;",
-                },
-            ),
-        }
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        config = dict(self.instance.config or {}) if self.instance and self.instance.pk else {}
-        if self.instance and self.instance.pk:
-            self.initial["tags_editor"] = self.instance.tags_str
-        effective_only_new = self.effective_only_new(self.instance if self.instance and self.instance.pk else None)
-        derived_filter_toggles = self.derive_filter_toggles(
-            self.instance.urls if self.instance and self.instance.pk else "",
-            config.get("URL_ALLOWLIST", ""),
-        )
-        self.initial["url_filters"] = {
-            "allowlist": config.get("URL_ALLOWLIST", ""),
-            "denylist": config.get("URL_DENYLIST", ""),
-            "same_domain_only": derived_filter_toggles["same_domain_only"],
-            "subpaths_only": derived_filter_toggles["subpaths_only"],
-            "only_new": effective_only_new,
-        }
-
-    @staticmethod
-    def extract_url_line(line):
-        line = str(line or "").strip()
-        if not line or line.startswith("#"):
-            return ""
-        if line.startswith("{"):
-            try:
-                return str(json.loads(line).get("url", "")).strip()
-            except (TypeError, ValueError, json.JSONDecodeError):
-                return ""
-        return line
-
-    @staticmethod
-    def regex_escape(text):
-        escaped = ""
-        for char in str(text or ""):
-            escaped += f"\\{char}" if char in r".*+?^${}()|[]\\" else char
-        return escaped
-
-    @classmethod
-    def generated_host_allowlist(cls, urls):
-        seen = set()
-        domains = []
-        for raw_line in str(urls or "").splitlines():
-            url = cls.extract_url_line(raw_line)
-            if not url:
-                continue
-            parsed = urlparse(url)
-            domain = (parsed.hostname or "").lower()
-            if not domain or domain in seen:
-                continue
-            seen.add(domain)
-            domains.append(domain)
-        if not domains:
-            return ""
-        return "^https?://(" + "|".join(cls.regex_escape(domain) for domain in domains) + ")([:/]|$)"
-
-    @staticmethod
-    def subpath_prefix(pathname):
-        path = str(pathname or "/")
-        while "//" in path:
-            path = path.replace("//", "/")
-        if not path or path == "/":
-            return "/"
-        if path.endswith("/"):
-            return path
-        last_slash = path.rfind("/")
-        last_part = path[last_slash + 1 :]
-        if "." in last_part:
-            return path[: last_slash + 1] or "/"
-        return path
-
-    @staticmethod
-    def parsed_host_and_port(parsed):
-        host = (parsed.hostname or "").lower()
-        if not host:
-            return ""
-        try:
-            port = parsed.port
-        except ValueError:
-            port = None
-        return f"{host}:{port}" if port is not None else host
-
-    @classmethod
-    def generated_subpath_allowlist(cls, urls):
-        seen = set()
-        paths = []
-        for raw_line in str(urls or "").splitlines():
-            url = cls.extract_url_line(raw_line)
-            if not url:
-                continue
-            parsed = urlparse(url)
-            domain = (parsed.hostname or "").lower()
-            if domain:
-                seen.add(domain)
-            host = cls.parsed_host_and_port(parsed)
-            path = cls.subpath_prefix(parsed.path)
-            path_key = f"{host}{path}"
-            if not host or path_key in seen:
-                continue
-            seen.add(path_key)
-            paths.append((host, path))
-        if not paths:
-            return ""
-        patterns = []
-        for host, path in paths:
-            if path == "/":
-                patterns.append(f"^https?://{cls.regex_escape(host)}([/?#]|$)")
-            elif path.endswith("/"):
-                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}")
-            else:
-                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}([/?#]|$)")
-        return "\n".join(patterns)
-
-    @classmethod
-    def derive_filter_toggles(cls, urls, allowlist):
-        normalized_allowlist = "\n".join(Crawl.split_filter_patterns(allowlist))
-        if not normalized_allowlist:
-            return {"same_domain_only": False, "subpaths_only": False}
-        if normalized_allowlist == cls.generated_subpath_allowlist(urls):
-            return {"same_domain_only": True, "subpaths_only": True}
-        if normalized_allowlist == cls.generated_host_allowlist(urls):
-            return {"same_domain_only": True, "subpaths_only": False}
-        return {"same_domain_only": False, "subpaths_only": False}
-
-    @staticmethod
-    def effective_only_new(crawl=None):
-        from archivebox.config.common import get_config
-
-        if crawl is not None:
-            return bool(get_config(crawl=crawl, resolve_plugins=False).ONLY_NEW)
-        return bool(get_config(resolve_plugins=False).ONLY_NEW)
-
-    @staticmethod
-    def inherited_only_new(crawl):
-        crawl_without_only_new = copy(crawl)
-        config = dict(crawl.config or {})
-        config.pop("ONLY_NEW", None)
-        crawl_without_only_new.config = config
-        return CrawlAdminForm.effective_only_new(crawl_without_only_new)
-
-    def clean_tags_editor(self):
-        tags_str = self.cleaned_data.get("tags_editor", "")
-        tag_names = []
-        seen = set()
-        for raw_name in tags_str.split(","):
-            name = raw_name.strip()
-            if not name:
-                continue
-            lowered = name.lower()
-            if lowered in seen:
-                continue
-            seen.add(lowered)
-            tag_names.append(name)
-        return ",".join(tag_names)
-
-    def save(self, commit=True):
-        instance = super().save(commit=False)
-        instance.tags_str = self.cleaned_data.get("tags_editor", "")
-        if f"{self.add_prefix('url_filters')}_allowlist" in self.data or f"{self.add_prefix('url_filters')}_denylist" in self.data:
-            url_filters = self.cleaned_data.get("url_filters") or {}
-            instance.set_url_filters(
-                url_filters.get("allowlist", ""),
-                url_filters.get("denylist", ""),
-            )
-            config = dict(instance.config or {})
-            only_new = bool(url_filters.get("only_new"))
-            inherited_only_new = self.inherited_only_new(instance)
-            if only_new != inherited_only_new:
-                config["ONLY_NEW"] = only_new
-            else:
-                config.pop("ONLY_NEW", None)
-            instance.config = config
-        if commit:
-            instance.save()
-            instance.apply_crawl_config_filters()
-            self._save_m2m()
-        return instance

archivebox/crawls/models.py

@@ -1,6 +1,6 @@
 __package__ = "archivebox.crawls"
 
-from typing import TYPE_CHECKING, Any, ClassVar
+from typing import TYPE_CHECKING, Any
 from collections.abc import Iterable, Mapping
 import uuid
 import json
@@ -31,7 +31,7 @@ from archivebox.base_models.models import (
     ModelWithHealthStats,
     get_or_create_system_user_pk,
 )
-from archivebox.workers.models import DefaultStatusChoices, ModelWithQueue
+from archivebox.workers.models import ModelWithQueue
 from archivebox.crawls.schedule_util import next_run_for_schedule, validate_schedule
 from archivebox.misc.util import parse_date, sanitize_html_text, validate_url, validate_url_length
 
@@ -184,7 +184,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     retry_at_field_name = "retry_at"
     state_field_name = "status"
-    StatusChoices: ClassVar[type[DefaultStatusChoices]] = DefaultStatusChoices
+    StatusChoices = ModelWithQueue.StatusChoices
     INITIAL_STATE = StatusChoices.QUEUED
     ACTIVE_STATE = StatusChoices.STARTED
     FINAL_STATES = (StatusChoices.SEALED,)
@@ -467,6 +467,8 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             "tags_str": self.tags_str,
             "label": self.label,
             "created_at": self.created_at.isoformat() if self.created_at else None,
+            "modified_at": self.modified_at.isoformat() if self.modified_at else None,
+            "created_by": self.created_by.username,
         }
 
     @staticmethod
@@ -844,24 +846,31 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         output_dir: Path | None = None,
         num_snapshots: int | None = None,
     ) -> str:
-        from abx_dl.limits import CrawlLimitState
+        from abx_dl.limits import parse_filesize_to_bytes
 
-        if output_dir is None:
-            output_dir = self.output_dir
         if config is None:
             from archivebox.config.common import get_config
 
-            config = get_config(crawl=self, include_machine=False).for_crawl_runtime(
-                crawl=self,
-                persona=self.resolve_persona(),
-                crawl_output_dir=output_dir,
-            )
+            config = get_config(crawl=self, include_machine=False)
 
-        limits_path = output_dir / ".abx-dl" / "limits.json"
-        if limits_path.exists():
-            stop_reason = CrawlLimitState.from_config(config).get_stop_reason()
-            if stop_reason:
-                return stop_reason
+        # The DB is the shared authority for workers and the UI. A second
+        # ledger beside the artifacts can disagree after cancellation/retry and
+        # would require filesystem access just to render a crawl's status.
+        # Recompute against the current limits so increasing a cap permits work.
+        timeout = int(self._config_value(config, "CRAWL_TIMEOUT", 0) or 0)
+        if timeout:
+            # Snapshot creation already supplies a durable crawl clock. Using
+            # that existing fact avoids another timestamp to keep in sync and
+            # survives cancellation, which deliberately removes unfinished
+            # ArchiveResult rows while preserving the Snapshot for retry.
+            first_snapshot_time = self.snapshot_set.aggregate(first=models.Min("created_at"))["first"]
+            if first_snapshot_time and (timezone.now() - first_snapshot_time).total_seconds() >= timeout:
+                return "crawl_timeout"
+        max_size = parse_filesize_to_bytes(self._config_value(config, "CRAWL_MAX_SIZE", 0) or 0)
+        if max_size:
+            total_size = self.snapshot_set.aggregate(total=models.Sum("output_size"))["total"] or 0
+            if total_size >= max_size:
+                return "crawl_max_size"
 
         max_urls = int(self._config_value(config, "CRAWL_MAX_URLS", 0) or 0)
         if num_snapshots is None:
@@ -1248,10 +1257,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             }
             if tag_names:
                 tag_names_by_url[snapshot.url] = tag_names
-            # Snapshot.save() handles model-level validation. The crawl symlink
-            # can still wait until after commit so SQLite does not hold a write
-            # lock while touching the filesystem.
-            transaction.on_commit(lambda snapshot=snapshot: snapshot.ensure_crawl_symlink())
 
         tag_names = {tag for tags in tag_names_by_url.values() for tag in tags}
         if tag_names:

archivebox/machine/admin.py

@@ -1,5 +1,6 @@
 __package__ = "archivebox.machine"
 
+import json
 import shlex
 from pathlib import Path
 
@@ -13,13 +14,50 @@ from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django_object_actions import action
 
-from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
-from archivebox.core.widgets import render_copy_block
+from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 from archivebox.machine.env_util import env_to_dotenv_text
 from archivebox.machine.models import Binary, Machine, NetworkInterface, Process
 from archivebox.misc.logging_util import printable_filesize
 
 
+def _render_copy_block(text: str, *, multiline: bool = False):
+    if multiline:
+        return format_html(
+            """
+            <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
+                <button type="button"
+                        data-command="{}"
+                        onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
+                        style="position: absolute; top: 6px; right: 6px; z-index: 1; padding: 2px 8px; border: 0; border-radius: 4px; background: #e2e8f0; color: #334155; font-size: 11px; cursor: pointer;">
+                    Copy
+                </button>
+                <pre title="{}" style="display: block; width: 100%; max-width: 100%; overflow: auto; max-height: 300px; margin: 0; padding: 8px 56px 8px 8px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; font-size: 11px; line-height: 1.45; white-space: pre-wrap; word-break: break-word; box-sizing: border-box;">{}</pre>
+            </div>
+            """,
+            text,
+            text,
+            text,
+        )
+    return format_html(
+        """
+        <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
+            <button type="button"
+                    data-command="{}"
+                    onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
+                    style="position: absolute; top: 6px; right: 6px; z-index: 1; padding: 2px 8px; border: 0; border-radius: 4px; background: #e2e8f0; color: #334155; font-size: 11px; cursor: pointer;">
+                Copy
+            </button>
+            <code title="{}" style="display: block; width: 100%; max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 8px 56px 8px 8px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; font-size: 11px; box-sizing: border-box;">
+                {}
+            </code>
+        </div>
+        """,
+        text,
+        text,
+        text,
+    )
+
+
 def _format_process_duration_seconds(started_at, ended_at) -> str:
     if not started_at:
         return "-"
@@ -84,27 +122,59 @@ class MachineAdmin(ConfigEditorMixin, BaseModelAdmin):
     readonly_fields = ("guid", "created_at", "modified_at", "ips")
 
     fieldsets = (
-        card_fieldset("Identity", ("hostname", "guid", "ips")),
-        card_fieldset("Hardware", ("hw_manufacturer", "hw_product", "hw_uuid", "hw_in_docker", "hw_in_vm")),
-        card_fieldset("Operating System", ("os_platform", "os_family", "os_arch", "os_kernel", "os_release")),
-        card_fieldset("Statistics", ("stats", "num_uses_succeeded", "num_uses_failed")),
-        card_fieldset(
+        (
+            "Identity",
+            {
+                "fields": ("hostname", "guid", "ips"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Hardware",
+            {
+                "fields": ("hw_manufacturer", "hw_product", "hw_uuid", "hw_in_docker", "hw_in_vm"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Operating System",
+            {
+                "fields": ("os_platform", "os_family", "os_arch", "os_kernel", "os_release"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Statistics",
+            {
+                "fields": ("stats", "num_uses_succeeded", "num_uses_failed"),
+                "classes": ("card",),
+            },
+        ),
+        (
             "Configuration",
-            ("config",),
-            wide=True,
-            description=mark_safe(
-                '<div style="padding:8px 10px;margin-bottom:8px;background:#fff7ed;'
-                "border:1px solid #fed7aa;border-left:4px solid #f59e0b;border-radius:4px;"
-                'color:#7c2d12;font-size:12px;line-height:1.45;">'
-                "<b>Heads up:</b> saving here also rewrites "
-                "<code>data/ArchiveBox.conf</code> on disk to match — the two stores are "
-                "kept in 1:1 sync, so any keys you remove here will be removed from the file "
-                "too. Edits to <code>ArchiveBox.conf</code> (or <code>archivebox config --set</code>) "
-                "propagate back into this field on the next request."
-                "</div>",
-            ),
+            {
+                "fields": ("config",),
+                "classes": ("card", "wide"),
+                "description": mark_safe(
+                    '<div style="padding:8px 10px;margin-bottom:8px;background:#fff7ed;'
+                    "border:1px solid #fed7aa;border-left:4px solid #f59e0b;border-radius:4px;"
+                    'color:#7c2d12;font-size:12px;line-height:1.45;">'
+                    "<b>Heads up:</b> saving here also rewrites "
+                    "<code>data/ArchiveBox.conf</code> on disk to match — the two stores are "
+                    "kept in 1:1 sync, so any keys you remove here will be removed from the file "
+                    "too. Edits to <code>ArchiveBox.conf</code> (or <code>archivebox config --set</code>) "
+                    "propagate back into this field on the next request."
+                    "</div>",
+                ),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": ("created_at", "modified_at"),
+                "classes": ("card",),
+            },
         ),
-        card_fieldset("Timestamps", ("created_at", "modified_at")),
     )
 
     list_filter = ("hw_in_docker", "hw_in_vm", "os_arch", "os_family", "os_platform")
@@ -120,6 +190,12 @@ class MachineAdmin(ConfigEditorMixin, BaseModelAdmin):
             ", ".join(machine.networkinterface_set.values_list("ip_public", flat=True)),
         )
 
+    @admin.display(description="Health", ordering="health")
+    def health_display(self, obj):
+        h = obj.health
+        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
+        return format_html('<span style="color: {};">{}</span>', color, h)
+
     @admin.display(description="ID", ordering="id")
     def id_display(self, machine):
         # Highlight the row representing the machine that ``Machine.current()``
@@ -199,11 +275,41 @@ class NetworkInterfaceAdmin(BaseModelAdmin):
     readonly_fields = ("machine", "created_at", "modified_at", "mac_address", "ip_public", "ip_local", "dns_server")
 
     fieldsets = (
-        card_fieldset("Machine", ("machine",)),
-        card_fieldset("Network", ("iface", "ip_public", "ip_local", "mac_address", "dns_server")),
-        card_fieldset("Location", ("hostname", "isp", "city", "region", "country")),
-        card_fieldset("Usage", ("num_uses_succeeded", "num_uses_failed")),
-        card_fieldset("Timestamps", ("created_at", "modified_at")),
+        (
+            "Machine",
+            {
+                "fields": ("machine",),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Network",
+            {
+                "fields": ("iface", "ip_public", "ip_local", "mac_address", "dns_server"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Location",
+            {
+                "fields": ("hostname", "isp", "city", "region", "country"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Usage",
+            {
+                "fields": ("num_uses_succeeded", "num_uses_failed"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": ("created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
     )
 
     list_filter = ("isp", "country", "region")
@@ -220,6 +326,12 @@ class NetworkInterfaceAdmin(BaseModelAdmin):
             iface.machine.hostname,
         )
 
+    @admin.display(description="Health", ordering="health")
+    def health_display(self, obj):
+        h = obj.health
+        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
+        return format_html('<span style="color: {};">{}</span>', color, h)
+
 
 class BinaryAdmin(BaseModelAdmin):
     list_display = ("id", "created_at", "machine_info", "name", "binprovider", "version", "abspath", "sha256", "status", "health_display")
@@ -229,12 +341,48 @@ class BinaryAdmin(BaseModelAdmin):
     readonly_fields = ("created_at", "modified_at", "output_dir")
 
     fieldsets = (
-        card_fieldset("Binary Info", ("name", "binproviders", "binprovider", "overrides")),
-        card_fieldset("Location", ("machine", "abspath")),
-        card_fieldset("Version", ("version", "sha256")),
-        card_fieldset("State", ("status", "retry_at", "output_dir")),
-        card_fieldset("Usage", ("num_uses_succeeded", "num_uses_failed")),
-        card_fieldset("Timestamps", ("created_at", "modified_at")),
+        (
+            "Binary Info",
+            {
+                "fields": ("name", "binproviders", "binprovider", "overrides"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Location",
+            {
+                "fields": ("machine", "abspath"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Version",
+            {
+                "fields": ("version", "sha256"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "State",
+            {
+                "fields": ("status", "retry_at", "output_dir"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Usage",
+            {
+                "fields": ("num_uses_succeeded", "num_uses_failed"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": ("created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
     )
 
     list_filter = ("name", "binprovider", "status", "machine_id")
@@ -251,6 +399,12 @@ class BinaryAdmin(BaseModelAdmin):
             binary.machine.hostname,
         )
 
+    @admin.display(description="Health", ordering="health")
+    def health_display(self, obj):
+        h = obj.health
+        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
+        return format_html('<span style="color: {};">{}</span>', color, h)
+
 
 class ProcessAdmin(BaseModelAdmin):
     list_display = (
@@ -309,10 +463,34 @@ class ProcessAdmin(BaseModelAdmin):
     )
 
     fieldsets = (
-        card_fieldset("Process Info", ("machine", "archiveresult_link", "snapshot_link", "crawl_link", "status", "retry_at")),
-        card_fieldset("Command", ("cmd_display", "pwd", "env_display", "timeout"), wide=True),
-        card_fieldset("Execution", ("binary_link", "iface_link", "pid", "exit_code", "url")),
-        card_fieldset("Timing", ("started_at", "ended_at", "duration_display")),
+        (
+            "Process Info",
+            {
+                "fields": ("machine", "archiveresult_link", "snapshot_link", "crawl_link", "status", "retry_at"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Command",
+            {
+                "fields": ("cmd_display", "pwd", "env_display", "timeout"),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Execution",
+            {
+                "fields": ("binary_link", "iface_link", "pid", "exit_code", "url"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timing",
+            {
+                "fields": ("started_at", "ended_at", "duration_display"),
+                "classes": ("card",),
+            },
+        ),
         (
             "Output",
             {
@@ -320,7 +498,13 @@ class ProcessAdmin(BaseModelAdmin):
                 "classes": ("card", "wide", "collapse"),
             },
         ),
-        card_fieldset("Timestamps", ("created_at", "modified_at")),
+        (
+            "Timestamps",
+            {
+                "fields": ("created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
     )
 
     list_filter = ("status", "exit_code", "machine_id")
@@ -519,9 +703,36 @@ class ProcessAdmin(BaseModelAdmin):
     @admin.display(description="Output", ordering="archiveresult__output_size")
     def output_summary(self, process):
         try:
-            file_count, total_bytes = process.archiveresult.output_file_stats()
+            output_files = process.archiveresult.output_files or {}
         except Process.archiveresult.RelatedObjectDoesNotExist:
-            file_count, total_bytes = 0, 0
+            output_files = {}
+
+        if isinstance(output_files, str):
+            try:
+                output_files = json.loads(output_files)
+            except (json.JSONDecodeError, TypeError, ValueError):
+                output_files = {}
+
+        file_count = 0
+        total_bytes = 0
+
+        if isinstance(output_files, dict):
+            file_count = len(output_files)
+            items = output_files.values()
+        elif isinstance(output_files, (list, tuple, set)):
+            file_count = len(output_files)
+            items = output_files
+        else:
+            items = ()
+
+        for metadata in items:
+            if not isinstance(metadata, dict):
+                continue
+            size = metadata.get("size", 0)
+            try:
+                total_bytes += int(size or 0)
+            except (TypeError, ValueError):
+                continue
 
         file_label = "file" if file_count == 1 else "files"
         return format_html(
@@ -539,26 +750,26 @@ class ProcessAdmin(BaseModelAdmin):
             cmd = shlex.join(str(arg) for arg in process.cmd)
         else:
             cmd = str(process.cmd)
-        return render_copy_block(cmd)
+        return _render_copy_block(cmd)
 
     @admin.display(description="Environment")
     def env_display(self, process):
         env_text = env_to_dotenv_text(process.env)
         if not env_text:
             return "-"
-        return render_copy_block(env_text, multiline=True)
+        return _render_copy_block(env_text, multiline=True)
 
     @admin.display(description="Stdout")
     def stdout_display(self, process):
         if not process.stdout:
             return "-"
-        return render_copy_block(process.stdout, multiline=True)
+        return _render_copy_block(process.stdout, multiline=True)
 
     @admin.display(description="Stderr")
     def stderr_display(self, process):
         if not process.stderr:
             return "-"
-        return render_copy_block(process.stderr, multiline=True)
+        return _render_copy_block(process.stderr, multiline=True)
 
     @admin.display(description="ArchiveResult Output")
     def archiveresult_output_display(self, process):
@@ -568,7 +779,7 @@ class ProcessAdmin(BaseModelAdmin):
             return "-"
         if not output:
             return "-"
-        return render_copy_block(output, multiline=True)
+        return _render_copy_block(output, multiline=True)
 
 
 def register_admin(admin_site):

archivebox/machine/detect.py

@@ -88,7 +88,7 @@ def get_vm_info():
         try:
             # Getting SMBIOS data from sysfs.
             # SMBIOS 2.8 present.
-            # argo-1        | 2024-10-01T10:40:51Z ERR  error="Incoming request ended abruptly: context canceled" connIndex=2 event=1 ingressRule=0 originService=http://archivebox:8000                                                                               │
+            # argo-1        | 2024-10-01T10:40:51Z ERR  error="Incoming request ended abruptly: context canceled" connIndex=2 event=1 ingressRule=0 originService=http://archivebox:5797                                                                               │
             # Handle 0x0100, DMI type 1, 27 bytes
             # System Information
             #         Manufacturer: DigitalOcean
@@ -217,27 +217,26 @@ def get_isp_info(ip=None):
     }
 
 
-def get_local_interface(address: str) -> dict[str, str]:
-    """Describe the real host interface owning an IPv4 address."""
-    for interface, addresses in psutil.net_if_addrs().items():
-        if any(ip.address == address for ip in ip_addrs(addresses)):
-            return {
-                "iface": interface,
-                "mac_address": next((mac.address for mac in mac_addrs(addresses)), ""),
-                "ip_local": address,
-            }
-    raise ValueError(f"No host interface owns {address}")
-
-
 def get_host_network() -> dict[str, Any]:
-    interface = get_local_interface(get_local_ip())
-    public_ip = get_public_ip()
-    return {
-        "hostname": max([socket.gethostname(), platform.node()], key=len),
-        **interface,
-        "ip_public": public_ip,
-        **get_isp_info(public_ip),
-    }
+    default_gateway_local_ip = get_local_ip()
+    gateways = psutil.net_if_addrs()
+
+    for interface, ips in gateways.items():
+        for local_ip in ip_addrs(ips):
+            if default_gateway_local_ip == local_ip.address:
+                mac_address = next(mac_addrs(ips)).address
+                public_ip = get_public_ip()
+                return {
+                    "hostname": max([socket.gethostname(), platform.node()], key=len),
+                    "iface": interface,
+                    "mac_address": mac_address,
+                    "ip_local": local_ip.address,
+                    "ip_public": public_ip,
+                    # "is_behind_nat": local_ip.address != public_ip,
+                    **get_isp_info(public_ip),
+                }
+
+    raise Exception("Could not determine host network info")
 
 
 def get_os_info() -> dict[str, Any]: