Bump release version to 0.9.35rc458

.github/workflows/deploy-publicsite.yml

@@ -10,6 +10,7 @@ on:
       - bin/collect_ui_screenshots.*
       - bin/generate_ui_screenshot_gallery.py
       - bin/take_screenshot.js
+      - docs/Screenshots.md
       - pyproject.toml
       - uv.lock
       - .github/workflows/deploy-publicsite.yml

.pre-commit-config.yaml

@@ -64,4 +64,4 @@ repos:
         args: ["--maxkb=600"]
       - id: name-tests-test
         args: ["--pytest-test-first"]
-        exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$|import_helpers\.py$)
+        exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$)

Dockerfile

@@ -10,10 +10,10 @@
 #       --build-context abx-plugins=../abx-plugins \
 #       -t archivebox/abx-dl:dev
 #   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.274 \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.279 \
 #       -t archivebox:multistage
 
-ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.274
+ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.279
 
 FROM archivebox/sonic:1.4.9 AS sonic
 FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
@@ -96,17 +96,15 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
     <<'EOF'
 echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
-echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache
-echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends
-echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests
+export UV_NO_CACHE=true
 rm -f /etc/apt/apt.conf.d/docker-clean
 apt-get update -qq
-apt-get install -qq -y --no-install-recommends \
-    build-essential gcc libldap2-dev libsasl2-dev libssl-dev x11-utils
+apt-get install -qq -y --no-install-recommends x11-utils
 # Extend abx-dl's existing venv instead of clearing it. Clearing and later
 # copying a complete replacement over the base duplicates every venv byte in
 # the final overlay history even when most packages are unchanged.
-/usr/bin/uv pip install --no-cache setuptools pip wheel
+# Do not rewrite inherited native libraries merely to strip them again.
+/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print > /tmp/archivebox-inherited-libraries
 
 mkdir -p /tmp/archivebox-uv-project
 /venv/bin/python - <<'PY'
@@ -155,25 +153,31 @@ PY
     --inexact \
     --no-install-project \
     --no-install-workspace \
+    --no-install-package abxbus \
+    --no-install-package abxpkg \
+    --no-install-package abx-plugins \
+    --no-install-package abx-dl \
     --no-sources
 builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
 ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
 /usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
 while IFS= read -r -d '' native_library; do
+    if grep -Fxq "$native_library" /tmp/archivebox-inherited-libraries; then
+        continue
+    fi
     magic=''
     if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then
         "$builder_abxpkg_lib_dir/env/bin/strip" --strip-unneeded "$native_library" || exit $?
     fi
 done < /tmp/archivebox-native-libraries
-rm -f /tmp/archivebox-native-libraries
+rm -f /tmp/archivebox-native-libraries /tmp/archivebox-inherited-libraries
 rm -f /venv/bin/uv /venv/bin/uvx
-abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get purge -y binutils build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get purge -y binutils
 abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get autoremove -y
 rm -rf "$builder_abxpkg_lib_dir"
 rm -rf /venv/lib/python3.*/site-packages/pip* \
     /venv/lib/python3.*/site-packages/wheel* \
     /venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel
-echo 'Binary::apt::APT::Keep-Downloaded-Packages "0";' > /etc/apt/apt.conf.d/99keep-cache
 rm -rf /var/lib/apt/lists/* /tmp/archivebox-uv-project
 EOF
 
@@ -202,7 +206,7 @@ RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
     && test -f "$(/venv/bin/python -c 'import archivebox; print(archivebox.__cached__)')" \
     && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
 
-# The builder installs and purges compilers in one layer, so its final
+# The builder installs and purges stripping tools in one layer, so its final
 # filesystem is already runtime-clean. Preserve that ancestry: starting again
 # from archivebox-runtime-base and COPYing /venv would bake a second full venv
 # over the inherited one instead of recording only ArchiveBox's package delta.
@@ -225,7 +229,8 @@ COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
 # The builder invokes abxpkg as root for temporary ELF stripping, which can
 # rewrite derived state ownership. Restore UID 911 before resolving binaries
 # as the runtime user; the old copy-based final stage hid this dependency.
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
+RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777,id=archivebox-runtime-config-$TARGETARCH \
+    echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
     && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
     && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
     && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
@@ -239,8 +244,8 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..."
     && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
     && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
     && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
-    && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$ABXPKG_LIB_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
+    && find "$ABXPKG_LIB_DIR" \( ! -user "$DEFAULT_ARCHIVEBOX_UID" -o ! -group "$DEFAULT_ARCHIVEBOX_GID" \) -exec chown -h "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" {} + \
+    && UV_NO_CACHE=true setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
     && openssl rand -hex 16 > /etc/machine-id \
     && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
     && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt

README.md

@@ -1,5 +1,5 @@
 <div align="center" style="text-align: center; width: 100%">
-<img style="height: 90px" src="https://archivebox.io/icon.png"/>
+<img src="https://archivebox.io/icon.png" height="90px"/>
 <h1>ArchiveBox<br/><sub>Open-source self-hosted web archiving.</sub></h1>
 
 <br/>
@@ -108,7 +108,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 
 <div align="center" style="text-align: center">
 <br/><br/>
-<img style="height: 70px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" alt="bookshelf graphic"> &nbsp; <img style="height: 75px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/b2765a33-0d1e-4019-a1db-920c7e00e20e" alt="logo" align="top"/> &nbsp; <img style="height: 70px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" alt="bookshelf graphic">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic"> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/b2765a33-0d1e-4019-a1db-920c7e00e20e" height="75px" alt="logo" align="top"/> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic">
 <br/><br/>
 <small><a href="https://demo.archivebox.io">Demo</a> | <a href="#screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
 <br/>
@@ -152,7 +152,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 #### ✳️&nbsp; Easy Setup
 
 <details>
-<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <code>docker-compose</code></b>  (macOS/Linux) &nbsp; <b>👈&nbsp; recommended</b> &nbsp; <i>(click to expand)</i></summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker-compose</code></b>  (macOS/Linux) &nbsp; <b>👈&nbsp; recommended</b> &nbsp; <i>(click to expand)</i></summary>
 <br/>
 <i>👍 Docker Compose is recommended for the easiest install/update UX + best security + all <a href="#dependencies">extras</a> out-of-the-box.</i>
 <br/><br/>
@@ -181,7 +181,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 </details>
 
 <details>
-<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <code>docker run</code></b>  (macOS/Linux)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker run</code></b>  (macOS/Linux)</summary>
 <br/>
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
@@ -205,7 +205,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 </details>
 
 <details>
-<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh uv install shortcut" align="top"/> <code>bash</code> uv install shortcut</b>  (macOS/Linux/BSD)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh uv install shortcut" height="28px" align="top"/> <code>bash</code> uv install shortcut</b>  (macOS/Linux/BSD)</summary>
 <br/>
 <ol>
 <li>Run the shortcut for the <code>uv</code> install method. It installs <code>uv</code> first when needed.
@@ -228,7 +228,7 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a
 
 
 <details>
-<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="uv" align="top"/> <code>uv</code></b> (macOS/Linux/BSD)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="uv" height="28px" align="top"/> <code>uv</code></b> (macOS/Linux/BSD)</summary>
 <br/>
 <ol>
 
@@ -262,7 +262,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 
 
 <details>
-<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
 <ol>
 <li>Add the third-party ArchiveBox apt repo and install <code>archivebox</code>.
@@ -295,7 +295,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 </details>
 
 <details>
-<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447803-f2ec3700-af0b-11eb-87d3-671d114f011d.png" alt="homebrew" align="top"/> <code>brew</code></b> (macOS and Linux)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447803-f2ec3700-af0b-11eb-87d3-671d114f011d.png" alt="homebrew" height="28px" align="top"/> <code>brew</code></b> (macOS and Linux)</summary>
 <br/>
 Run Homebrew as your normal non-root user on both macOS and Linux; do not use <code>sudo brew</code>.
 <ol>
@@ -329,7 +329,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 </details>
 
 <details>
-<summary><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/118077361-f0616580-b381-11eb-973c-ee894a3349fb.png" alt="Arch" align="top"/> <code>pacman</code> / <img style="height: 28px" src="https://user-images.githubusercontent.com/511499/118077946-29e6a080-b383-11eb-94f0-d4871da08c3f.png" alt="FreeBSD" align="top"/> <code>pkg</code> / <img style="height: 28px" src="https://user-images.githubusercontent.com/511499/118077861-002d7980-b383-11eb-86a7-5936fad9190f.png" alt="Nix" align="top"/> <code>nix</code> (Arch/FreeBSD/NixOS/more)</summary>
+<summary><img src="https://user-images.githubusercontent.com/511499/118077361-f0616580-b381-11eb-973c-ee894a3349fb.png" alt="Arch" height="28px" align="top"/> <code>pacman</code> / <img src="https://user-images.githubusercontent.com/511499/118077946-29e6a080-b383-11eb-94f0-d4871da08c3f.png" alt="FreeBSD" height="28px" align="top"/> <code>pkg</code> / <img src="https://user-images.githubusercontent.com/511499/118077861-002d7980-b383-11eb-86a7-5936fad9190f.png" alt="Nix" height="28px" align="top"/> <code>nix</code> (Arch/FreeBSD/NixOS/more)</summary>
 <br/>
 
 > *Warning: These are contributed by external volunteers and may lag behind the official `uv` and Docker channels.*
@@ -350,7 +350,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 #### 🎗&nbsp; Other Options
 
 <details>
-<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <code>docker</code> + <img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447263-4316c980-af0b-11eb-928d-eaf1292ac646.png" alt="Electron" align="top"/> <code>electron</code> Desktop App</b> (macOS/Linux/Windows)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker</code> + <img src="https://user-images.githubusercontent.com/511499/117447263-4316c980-af0b-11eb-928d-eaf1292ac646.png" alt="Electron" height="28px" align="top"/> <code>electron</code> Desktop App</b> (macOS/Linux/Windows)</summary>
 <br/>
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
@@ -369,7 +369,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 </details>
 
 <details>
-<summary><img style="height: 28px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0c46e949-00fe-49c8-a613-ee14501c014c" alt="Self-hosting Platforms" align="top"/><b> TrueNAS / UNRAID / YunoHost / Cloudron / etc.</b> (self-hosting solutions)</summary>
+<summary><img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0c46e949-00fe-49c8-a613-ee14501c014c" alt="Self-hosting Platforms" height="28px" align="top"/><b> TrueNAS / UNRAID / YunoHost / Cloudron / etc.</b> (self-hosting solutions)</summary>
 <br/>
 
 > *Warning: These are contributed by external volunteers and may lag behind the official Docker and `uv` channels.*
@@ -393,26 +393,26 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 </details>
 
 <details>
-<summary><img style="height: 27px" src="https://user-images.githubusercontent.com/511499/117448723-1663b180-af0d-11eb-837f-d43959227810.png" alt="paid" align="top"/> Third-party paid hosting options (cloud VPS)</summary>
+<summary><img src="https://user-images.githubusercontent.com/511499/117448723-1663b180-af0d-11eb-837f-d43959227810.png" alt="paid" height="27px" align="top"/> Third-party paid hosting options (cloud VPS)</summary>
 <br/>
 <ul>
-<li><a href="https://elest.io/open-source/archivebox"><img style="height: 22px" src="https://img.shields.io/badge/Managed_Hosting-Elest.io-%23193f7e.svg?style=flat"/></a></li>
-<li><a href="https://www.stellarhosted.com/archivebox/"><img style="height: 22px" src="https://img.shields.io/badge/Semi_Managed_Hosting-StellarHosted.com-%23193f7e.svg?style=flat"/></a> (USD $29-250/mo, <a href="https://www.stellarhosted.com/archivebox/#pricing">pricing</a>)</li>
-<li><a href="https://www.pikapods.com/pods?run=archivebox"><img style="height: 22px" src="https://img.shields.io/badge/Semi_Managed_Hosting-PikaPods.com-%2343a047.svg?style=flat"/></a> (from USD $2.6/mo)</li>
+<li><a href="https://elest.io/open-source/archivebox"><img src="https://img.shields.io/badge/Managed_Hosting-Elest.io-%23193f7e.svg?style=flat" height="22px"/></a></li>
+<li><a href="https://www.stellarhosted.com/archivebox/"><img src="https://img.shields.io/badge/Semi_Managed_Hosting-StellarHosted.com-%23193f7e.svg?style=flat" height="22px"/></a> (USD $29-250/mo, <a href="https://www.stellarhosted.com/archivebox/#pricing">pricing</a>)</li>
+<li><a href="https://www.pikapods.com/pods?run=archivebox"><img src="https://img.shields.io/badge/Semi_Managed_Hosting-PikaPods.com-%2343a047.svg?style=flat" height="22px"/></a> (from USD $2.6/mo)</li>
 <li><a href="https://m.do.co/c/cbc4c0c17840">
- <img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-DigitalOcean.com-%232f7cf7.svg?style=flat"/>
+ <img src="https://img.shields.io/badge/Unmanaged_VPS-DigitalOcean.com-%232f7cf7.svg?style=flat" height="22px"/>
 </a> (USD $5-50+/mo, <a href="https://m.do.co/c/cbc4c0c17840">🎗&nbsp; referral link</a>, <a href="https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-compose-on-ubuntu-20-04">instructions</a>)</li>
 <li><a href="https://www.vultr.com/?ref=7130289">
- <img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-Vultr.com-%232337a8.svg?style=flat"/>
+ <img src="https://img.shields.io/badge/Unmanaged_VPS-Vultr.com-%232337a8.svg?style=flat" height="22px"/>
 </a> (USD $2.5-50+/mo, <a href="https://www.vultr.com/?ref=7130289">🎗&nbsp; referral link</a>, <a href="https://www.vultr.com/docs/install-docker-compose-on-ubuntu-20-04">instructions</a>)</li>
 <li><a href="https://fly.io/">
- <img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_App-Fly.io-%239a2de6.svg?style=flat"/>
+ <img src="https://img.shields.io/badge/Unmanaged_App-Fly.io-%239a2de6.svg?style=flat" height="22px"/>
 </a> (USD $10-50+/mo, <a href="https://fly.io/docs/hands-on/start/">instructions</a>)</li>
 <li><a href="https://railway.app/template/2Vvhmy">
- <img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_App-Railway-%23A11BE6.svg?style=flat"/>
+ <img src="https://img.shields.io/badge/Unmanaged_App-Railway-%23A11BE6.svg?style=flat" height="22px"/>
 </a> (USD $0-5+/mo)</li>
-<li><a href="https://aws.amazon.com/marketplace/pp/Linnovate-Open-Source-Innovation-Support-For-Archi/B08RVW6MJ2"><img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-AWS-%23ee8135.svg?style=flat"/></a> (USD $60-200+/mo)</li>
-<li><a href="https://azuremarketplace.microsoft.com/en-us/marketplace/apps/meanio.archivebox?ocid=gtmrewards_whatsnewblog_archivebox_vol118"><img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-Azure-%237cb300.svg?style=flat"/></a> (USD $60-200+/mo)</li>
+<li><a href="https://aws.amazon.com/marketplace/pp/Linnovate-Open-Source-Innovation-Support-For-Archi/B08RVW6MJ2"><img src="https://img.shields.io/badge/Unmanaged_VPS-AWS-%23ee8135.svg?style=flat" height="22px"/></a> (USD $60-200+/mo)</li>
+<li><a href="https://azuremarketplace.microsoft.com/en-us/marketplace/apps/meanio.archivebox?ocid=gtmrewards_whatsnewblog_archivebox_vol118"><img src="https://img.shields.io/badge/Unmanaged_VPS-Azure-%237cb300.svg?style=flat" height="22px"/></a> (USD $60-200+/mo)</li>
 <br/>
 <sub><i>Referral links marked 🎗 provide $5-10 of free credit for new users and help pay for our <a href="https://demo.archivebox.io">demo server</a> hosting costs.</i></sub>
 </ul>
@@ -465,7 +465,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev help
 
 <br/>
 <details>
-<summary><img style="height: 22px" src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" align="top"/> <b>CLI Usage Examples: non-Docker</b></summary>
+<summary><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" height="22px" align="top"/> <b>CLI Usage Examples: non-Docker</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you have uv-installed ArchiveBox and it's available in your $PATH first
@@ -484,7 +484,7 @@ archivebox help              # get list of archivebox subcommands that can be ru
 <br/>
 
 <details>
-<summary><img style="height: 22px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <b>CLI Usage Examples: Docker Compose</b></summary>
+<summary><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="22px" align="top"/> <b>CLI Usage Examples: Docker Compose</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
@@ -501,7 +501,7 @@ docker compose exec archivebox archivebox add 'https://example.com'
 <br/>
 
 <details>
-<summary><img style="height: 22px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <b>CLI Usage Examples: Docker</b></summary>
+<summary><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="22px" align="top"/> <b>CLI Usage Examples: Docker</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure the `archivebox` server container from the Quickstart is running first
@@ -613,19 +613,19 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
 ##  Input Formats: How to pass URLs into ArchiveBox for saving
 
 
-- <img style="height: 28px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ff20d251-5347-4b85-ae9b-83037d0ac01e"/> <b>From the official <a href="https://github.com/ArchiveBox/archivebox-extension">ArchiveBox Browser Extension</a></b><br>
+- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ff20d251-5347-4b85-ae9b-83037d0ac01e" height="28px"/> <b>From the official <a href="https://github.com/ArchiveBox/archivebox-extension">ArchiveBox Browser Extension</a></b>  
   <i>Provides realtime archiving of browsing history or selected pages from Chrome/Chromium/Firefox browsers.</i>
 
-- <img style="height: 22px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files<br>
+- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8" height="22px"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files  
   <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file).</i>
 
-- <img style="height: 22px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)<br>
+- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34" height="22px"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)  
   <i>Instructions: <a href="https://support.google.com/chrome/answer/96816?hl=en">Chrome</a>, <a href="https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer">Firefox</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/assets/511499/24ad068e-0fa6-41f4-a7ff-4c26fc91f71a">Safari</a>, <a href="https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows">IE</a>, <a href="https://help.opera.com/en/latest/features/#bookmarks:~:text=Click%20the%20import/-,export%20button,-on%20the%20bottom">Opera</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
 
-- <img style="height: 22px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/4f7bd318-265c-4235-ad25-38be89946b12"/> From URLs visited through a [MITM Proxy](https://mitmproxy.org/) with [`archivebox-proxy`](https://github.com/ArchiveBox/archivebox-proxy)<br>
+- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/4f7bd318-265c-4235-ad25-38be89946b12" height="22px"/> From URLs visited through a [MITM Proxy](https://mitmproxy.org/) with [`archivebox-proxy`](https://github.com/ArchiveBox/archivebox-proxy)  
   <i>Provides [realtime archiving](https://github.com/ArchiveBox/ArchiveBox/issues/577) of all traffic from any device going through the proxy.</i>
 
-- <img style="height: 22px" src="https://getpocket.com/favicon.ico"/> From bookmarking services or social media (e.g. Twitter bookmarks, Reddit saved posts, etc.)<br>
+- <img src="https://getpocket.com/favicon.ico" height="22px"/> From bookmarking services or social media (e.g. Twitter bookmarks, Reddit saved posts, etc.)  
   <i>Instructions: <a href="https://getpocket.com/export">Pocket</a>, <a href="https://pinboard.in/export/">Pinboard</a>, <a href="https://www.instapaper.com/user">Instapaper</a>, <a href="https://shaarli.readthedocs.io/en/master/Usage/#importexport">Shaarli</a>, <a href="https://www.groovypost.com/howto/howto/export-delicious-bookmarks-xml/">Delicious</a>, <a href="https://github.com/csu/export-saved-reddit">Reddit Saved</a>, <a href="https://doc.wallabag.org/en/user/import/wallabagv2.html">Wallabag</a>, <a href="http://help.unmark.it/import-export">Unmark.it</a>, <a href="https://www.addictivetips.com/web/onetab-save-close-all-chrome-tabs-to-restore-export-or-import/">OneTab</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/issues/648">Firefox Sync</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
 
 
@@ -1009,7 +1009,7 @@ Each re-archive creates a distinct Snapshot row for the same URL:
 archivebox add --no-only-new 'https://example.com'
 </code></pre>
 
-The <img style="height: 24px" src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button"/> button in the Admin UI performs the same explicit re-archive.
+The <img src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button" height="24px"/> button in the Admin UI performs the same explicit re-archive.
 
 <h4>Learn More</h4>
 
@@ -1328,7 +1328,7 @@ archivebox server 0.0.0.0:8000
 
 ```bash
 # Optional: develop via docker by mounting the code dir into the container
-# if you edit e.g. ./archivebox/core/models/snapshots.py on the docker host, runserver
+# if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
 # inside the container will reload and pick up your changes
 ./bin/build_docker.sh dev
 

archivebox/api/admin.py

@@ -8,7 +8,7 @@ from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
 from signal_webhooks.settings import webhook_settings
 from signal_webhooks.utils import get_webhook_model, model_from_reference
 
-from archivebox.base_models.admin import card_fieldset, BaseModelAdmin
+from archivebox.base_models.admin import BaseModelAdmin
 
 from archivebox.api.models import APIToken
 
@@ -25,9 +25,27 @@ class APITokenAdmin(BaseModelAdmin):
     search_fields = ("id", "created_by__username", "token")
 
     fieldsets = (
-        card_fieldset("Token", ("token", "expires")),
-        card_fieldset("Owner", ("created_by",)),
-        card_fieldset("Timestamps", ("created_at", "modified_at")),
+        (
+            "Token",
+            {
+                "fields": ("token", "expires"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Owner",
+            {
+                "fields": ("created_by",),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": ("created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
     )
 
     list_filter = ("created_by",)
@@ -55,11 +73,41 @@ class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
     readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)
 
     fieldsets = (
-        card_fieldset("Webhook", _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"), wide=True),
-        card_fieldset("Authentication", _webhook_fields("auth_token")),
-        card_fieldset("Status", _webhook_fields("enabled", "last_success", "last_failure", "last_response")),
-        card_fieldset("Owner", _webhook_fields("created_by")),
-        card_fieldset("Timestamps", _webhook_fields("created_at", "modified_at")),
+        (
+            "Webhook",
+            {
+                "fields": _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Authentication",
+            {
+                "fields": _webhook_fields("auth_token"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Status",
+            {
+                "fields": _webhook_fields("enabled", "last_success", "last_failure", "last_response"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Owner",
+            {
+                "fields": _webhook_fields("created_by"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": _webhook_fields("created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
     )
 
     def lookup_allowed(self, lookup: str, value: str, request: HttpRequest | None = None) -> bool:

archivebox/api/schemas.py

@@ -1,17 +0,0 @@
-"""Fields shared by responses for user-owned model objects."""
-
-from ninja import Schema
-
-
-class OwnedObjectSchema(Schema):
-    created_by_id: str
-    created_by_username: str
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by.pk)
-
-    @staticmethod
-    def resolve_created_by_username(obj) -> str:
-        user = obj.created_by
-        return user.username if isinstance(user.username, str) else str(user)

archivebox/api/v1_auth.py

@@ -4,7 +4,7 @@ from django.http import HttpRequest
 
 from ninja import Router, Schema
 
-from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token
+from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token, HeaderTokenAuth, BearerTokenAuth
 
 
 router = Router(tags=["Authentication"], auth=None)
@@ -63,3 +63,44 @@ def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
         return {"success": True, "user_id": str(user.pk)}
 
     return {"success": False, "user_id": None}
+
+
+@router.post(
+    "/browser_session",
+    auth=[HeaderTokenAuth(), BearerTokenAuth()],
+    summary="Exchange an administrator API key for a normal browser session",
+)
+def browser_session(request: HttpRequest):
+    from django.conf import settings
+    from django.contrib.auth import login
+    from django.http import JsonResponse
+    from django.utils import timezone
+    from ninja.errors import HttpError
+    from archivebox.core.routes_util import get_admin_base_url
+
+    user = request.auth
+    if not user.is_active or not user.is_superuser:
+        raise HttpError(403, "An active administrator is required")
+    login(request, user, backend="django.contrib.auth.backends.ModelBackend")
+    # A browser session must not outlive the key used to establish it.
+    token = request._api_token
+    lifetime = settings.SESSION_COOKIE_AGE
+    if token.expires:
+        lifetime = min(lifetime, max(0, int((token.expires - timezone.now()).total_seconds())))
+    request.session.set_expiry(lifetime)
+    request.session.save()
+    response = JsonResponse(
+        {
+            "admin_url": get_admin_base_url(request=request).rstrip("/") + "/admin/",
+            "cookie": {
+                "name": settings.SESSION_COOKIE_NAME,
+                "value": request.session.session_key,
+                "expires": request.session.get_expiry_date().timestamp(),
+                "secure": settings.SESSION_COOKIE_SECURE,
+            },
+        },
+    )
+    # The native client installs a host-only HttpOnly cookie on the admin origin;
+    # API and admin hosts can differ, so a Set-Cookie on the API host is insufficient.
+    response["Cache-Control"] = "no-store"
+    return response

archivebox/api/v1_cli.py

@@ -117,27 +117,16 @@ def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_
     return kwargs
 
 
-def cli_response(request, result, *, result_format="json", success=True, errors=()):
-    """Use one response envelope for CLI-backed endpoints, including captured output."""
-    payload = {"success": success, "errors": list(errors), "result": result, "result_format": result_format}
-    for stream_name in ("stdout", "stderr"):
-        stream = request.__dict__.get(stream_name)
-        payload[stream_name] = ansi_to_html(stream.getvalue().strip()) if isinstance(stream, StringIO) else ""
-    return payload
-
-
-def command_config(args) -> dict[str, object] | None:
-    """Translate shared legacy add/schedule flags into runtime config overrides."""
-    if args.update or args.overwrite:
-        return {"ONLY_NEW": False}
-    return {"ONLY_NEW": bool(args.only_new)} if args.only_new is not None else None
-
-
 @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
 def cli_add(request: HttpRequest, args: AddCommandSchema):
     from archivebox.cli.archivebox_add import add
     from archivebox.misc.util import validate_url
 
+    config_overrides: dict[str, object] = {}
+    if args.only_new is not None:
+        config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
     submitted_urls: str | list[str] = args.urls
     if len(args.urls) == 1:
         try:
@@ -155,10 +144,20 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
             raise HttpError(400, "No valid URLs were submitted")
     crawl, snapshots = add(
         urls=submitted_urls,
-        **args.model_dump(exclude={"urls", "only_new", "update", "overwrite"}),
+        snapshot_ids=args.snapshot_ids,
+        tag=args.tag,
+        depth=args.depth,
+        max_urls=args.max_urls,
+        crawl_max_size=args.crawl_max_size,
+        crawl_timeout=args.crawl_timeout,
+        snapshot_max_size=args.snapshot_max_size,
+        index_only=args.index_only,
+        plugins=args.plugins,
+        persona=args.persona,
+        parser=args.parser,
         bg=True,  # Always run in background for API calls
         created_by_id=request.user.pk,
-        config=command_config(args),
+        config=config_overrides or None,
     )
 
     snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
@@ -168,7 +167,17 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         "snapshot_ids": snapshot_ids,
         "queued_urls": args.urls if isinstance(submitted_urls, str) else submitted_urls,
     }
-    return cli_response(request, result_payload, result_format="json")
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
+
+    return {
+        "success": True,
+        "errors": [],
+        "result": result_payload,
+        "result_format": "json",
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+    }
 
 
 @router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
@@ -195,20 +204,55 @@ def cli_update(request: HttpRequest, args: UpdateCommandSchema):
         ]
 
     update(**update_kwargs)
-    return cli_response(
-        request,
-        {"matched_count": len(matched_snapshot_ids), "snapshot_ids": matched_snapshot_ids} if is_filtered_update else None,
-        result_format="str",
-    )
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
+    return {
+        "success": True,
+        "errors": [],
+        "result": {
+            "matched_count": len(matched_snapshot_ids),
+            "snapshot_ids": matched_snapshot_ids,
+        }
+        if is_filtered_update
+        else None,
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+    }
 
 
 @router.post("/schedule", response=CLICommandResponseSchema, summary="archivebox schedule [args] [import_path]")
 def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
     from archivebox.cli.archivebox_schedule import schedule
 
-    result = schedule(**args.model_dump(exclude={"only_new", "update", "overwrite"}), config=command_config(args))
+    config_overrides: dict[str, object] = {}
+    if args.only_new is not None:
+        config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
+    result = schedule(
+        import_path=args.import_path,
+        add=args.add,
+        show=args.show,
+        foreground=args.foreground,
+        run_all=args.run_all,
+        quiet=args.quiet,
+        clear=args.clear,
+        every=args.every,
+        tag=args.tag,
+        depth=args.depth,
+        config=config_overrides or None,
+    )
 
-    return cli_response(request, result, result_format="json")
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "result_format": "json",
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+    }
 
 
 @router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
@@ -243,7 +287,16 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
     else:
         result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
 
-    return cli_response(request, result, result_format=result_format)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "result_format": result_format,
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+    }
 
 
 @router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
@@ -264,10 +317,13 @@ def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
         snapshots=snapshots_to_remove,
         timeout=timeout,
     )
-    return cli_response(
-        request,
-        result,
-        result_format="json",
-        success=bool(result["success"]),
-        errors=[str(result["error"])] if result["error"] else [],
-    )
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
+    return {
+        "success": bool(result["success"]),
+        "errors": [str(result["error"])] if result["error"] else [],
+        "result": result,
+        "result_format": "json",
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+    }

archivebox/api/v1_core/__init__.py

@@ -1,47 +0,0 @@
-"""Core REST routes, composed from entity-specific routers."""
-
-from typing import Union
-
-from django.db.models import Model
-from django.http import HttpRequest
-from django.shortcuts import redirect
-from ninja import Router
-from ninja.errors import HttpError
-
-from archivebox.api.v1_crawls import CrawlSchema, get_crawl
-
-from . import archiveresults, snapshots, tags
-from .archiveresults import get_archiveresult
-from .pagination import CustomPagination as CustomPagination
-from .schemas import ArchiveResultSchema, SnapshotSchema, TagSchema
-from .snapshots import get_snapshot
-from .tags import get_tag
-
-router = Router(tags=["Core Models"])
-router.add_router("", archiveresults.router)
-router.add_router("", snapshots.router)
-router.add_router("", tags.router)
-
-
-@router.get(
-    "/any/{id}",
-    response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema, CrawlSchema],
-    url_name="get_any",
-    summary="Get any object by its ID",
-)
-def get_any(request: HttpRequest, id: str):
-    """Get any object by its ID (e.g. snapshot, archiveresult, tag, crawl, etc.)."""
-    setattr(request, "with_snapshots", False)
-    setattr(request, "with_archiveresults", False)
-
-    for getter in (get_snapshot, get_archiveresult, get_tag, get_crawl):
-        try:
-            response = getter(request, id)
-            if isinstance(response, Model):
-                return redirect(
-                    f"/api/v1/{response._meta.app_label}/{response._meta.model_name}/{response.pk}?{request.META['QUERY_STRING']}",
-                )
-        except Exception:
-            pass
-
-    raise HttpError(404, "Object with given ID not found")

archivebox/api/v1_core/archiveresults.py

@@ -1,310 +0,0 @@
-import json
-import mimetypes
-import re
-from pathlib import Path, PurePosixPath
-from typing import Any
-
-from django.core.files.storage import FileSystemStorage
-from django.http import HttpRequest
-from django.http.multipartparser import MultiPartParser, MultiPartParserError
-from ninja import Form, Query, Router, UploadedFile
-from ninja.errors import HttpError
-from ninja.pagination import paginate
-
-from archivebox.core.models import ArchiveResult, Snapshot
-from archivebox.misc.db import uuid_ref_query as _uuid_ref_query
-
-from .lookup import _get_snapshot_by_ref
-from .pagination import CustomPagination
-from .schemas import ArchiveResultFilterSchema, ArchiveResultSchema
-
-router = Router(tags=["Core Models"])
-
-ARCHIVERESULT_UPLOAD_HOOK_NAME = Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME
-ARCHIVERESULT_UPLOAD_PLUGIN_RE = re.compile(r"^[A-Za-z0-9_.-]{1,32}$")
-
-
-@router.get("/archiveresults", response=list[ArchiveResultSchema], url_name="get_archiveresult")
-@paginate(CustomPagination)
-def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
-    """List all ArchiveResult entries matching these filters."""
-    queryset = filters.filter(ArchiveResult.objects.all())
-    if filters.search or filters.snapshot_tag:
-        return queryset.distinct()
-    return queryset
-
-
-@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
-def get_archiveresult(request: HttpRequest, archiveresult_id: str):
-    """Get a specific ArchiveResult by id."""
-    return ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
-
-
-def _normalize_uploaded_archiveresult_plugin(plugin: str) -> str:
-    normalized = str(plugin or "").strip().strip("/")
-    if not ARCHIVERESULT_UPLOAD_PLUGIN_RE.fullmatch(normalized):
-        raise HttpError(400, "Invalid ArchiveResult plugin name")
-    return normalized
-
-
-def _normalize_uploaded_archiveresult_output_path(output_path: str, *, filename: str) -> str:
-    raw_path = str(output_path or filename or "").strip().replace("\\", "/")
-    if not raw_path:
-        raise HttpError(400, "ArchiveResult output path is required")
-
-    path = PurePosixPath(raw_path)
-    if path.is_absolute() or any(part in ("", ".", "..") for part in path.parts):
-        raise HttpError(400, "Invalid ArchiveResult output path")
-
-    return str(path)
-
-
-def _parse_archiveresult_output_json(output_json: str | None) -> dict[str, Any] | None:
-    if not output_json:
-        return None
-    try:
-        parsed = json.loads(output_json)
-    except json.JSONDecodeError as err:
-        raise HttpError(400, "ArchiveResult output_json must be valid JSON") from err
-    if parsed is None:
-        return None
-    if not isinstance(parsed, dict):
-        raise HttpError(400, "ArchiveResult output_json must be a JSON object")
-    return parsed
-
-
-def _get_archiveresult_upload_data(request: HttpRequest):
-    cached = request.__dict__.get("_archiveresult_upload_data")
-    if cached is not None:
-        return cached
-
-    if request.method.upper() == "PATCH" and request.content_type.startswith("multipart/"):
-        try:
-            data = MultiPartParser(request.META, request, request.upload_handlers, request.encoding).parse()
-        except MultiPartParserError as err:
-            raise HttpError(400, f"Invalid ArchiveResult multipart upload: {err}") from err
-    else:
-        data = (request.POST, request.FILES)
-
-    setattr(request, "_archiveresult_upload_data", data)
-    return data
-
-
-def _get_archiveresult_upload_files(request: HttpRequest, *, allow_empty: bool = False) -> list[UploadedFile]:
-    _post, request_files = _get_archiveresult_upload_data(request)
-    files = [*request_files.getlist("files"), *request_files.getlist("file")]
-    if not files and not allow_empty:
-        raise HttpError(400, "At least one ArchiveResult file is required")
-    return files
-
-
-def _get_archiveresult_upload_form_values(request: HttpRequest, *field_names: str) -> list[str]:
-    request_post, _files = _get_archiveresult_upload_data(request)
-    values: list[str] = []
-    for field_name in field_names:
-        values.extend(str(value) for value in request_post.getlist(field_name))
-    if len(values) == 1:
-        value = values[0].strip()
-        if value.startswith("["):
-            try:
-                parsed = json.loads(value)
-            except json.JSONDecodeError:
-                parsed = None
-            if isinstance(parsed, list):
-                return [str(item) for item in parsed]
-    return values
-
-
-def _get_archiveresult_upload_form_value(request: HttpRequest, *field_names: str) -> str:
-    request_post, _files = _get_archiveresult_upload_data(request)
-    for field_name in field_names:
-        value = request_post.get(field_name)
-        if value is not None:
-            return str(value)
-    return ""
-
-
-def _parse_archiveresult_upload_int(value: str, field_name: str, *, default: int | None = None) -> int:
-    if value == "" and default is not None:
-        return default
-    try:
-        parsed = int(value)
-    except (TypeError, ValueError) as err:
-        raise HttpError(400, f"ArchiveResult {field_name} must be an integer") from err
-    if parsed < 0:
-        raise HttpError(400, f"ArchiveResult {field_name} must be non-negative")
-    return parsed
-
-
-def _write_archiveresult_files(
-    request: HttpRequest,
-    snapshot: Snapshot,
-    plugin_name: str,
-    *,
-    existing_output_files: dict[str, dict[str, Any]] | None = None,
-    allow_empty: bool = False,
-) -> dict[str, dict[str, Any]]:
-    files = _get_archiveresult_upload_files(request, allow_empty=allow_empty)
-    output_paths = _get_archiveresult_upload_form_values(request, "output_paths", "output_path")
-    mime_types = _get_archiveresult_upload_form_values(request, "mime_types", "mime_type")
-    chunk_output_path = _get_archiveresult_upload_form_value(request, "chunk_output_path")
-
-    snapshot_dir = snapshot.output_dir
-    plugin_dir = snapshot_dir / plugin_name
-    storage = FileSystemStorage(location=str(plugin_dir))
-    output_files = dict(existing_output_files or {})
-
-    if not files:
-        return output_files
-
-    if chunk_output_path:
-        if len(files) != 1:
-            raise HttpError(400, "Exactly one ArchiveResult file chunk is required")
-
-        uploaded_file = files[0]
-        relative_output_path = _normalize_uploaded_archiveresult_output_path(
-            chunk_output_path,
-            filename=uploaded_file.name,
-        )
-        chunk_index = _parse_archiveresult_upload_int(
-            _get_archiveresult_upload_form_value(request, "chunk_index"),
-            "chunk_index",
-        )
-        chunk_count = _parse_archiveresult_upload_int(
-            _get_archiveresult_upload_form_value(request, "chunk_count"),
-            "chunk_count",
-        )
-        chunk_offset = _parse_archiveresult_upload_int(
-            _get_archiveresult_upload_form_value(request, "chunk_offset"),
-            "chunk_offset",
-        )
-        chunk_total_size = _parse_archiveresult_upload_int(
-            _get_archiveresult_upload_form_value(request, "chunk_total_size"),
-            "chunk_total_size",
-        )
-
-        if chunk_count < 1:
-            raise HttpError(400, "ArchiveResult chunk_count must be at least 1")
-        if chunk_index >= chunk_count:
-            raise HttpError(400, "ArchiveResult chunk_index must be less than chunk_count")
-        if chunk_total_size and chunk_offset > chunk_total_size:
-            raise HttpError(400, "ArchiveResult chunk_offset cannot exceed chunk_total_size")
-
-        if chunk_index == 0 and chunk_offset == 0 and storage.exists(relative_output_path):
-            storage.delete(relative_output_path)
-
-        current_size = storage.size(relative_output_path) if storage.exists(relative_output_path) else 0
-        if current_size != chunk_offset:
-            raise HttpError(
-                409,
-                f"ArchiveResult chunk offset mismatch for {relative_output_path}: expected {current_size}, got {chunk_offset}",
-            )
-
-        Path(storage.path(relative_output_path)).parent.mkdir(parents=True, exist_ok=True)
-        with storage.open(relative_output_path, "ab") as destination:
-            for chunk in uploaded_file.chunks():
-                destination.write(chunk)
-
-        size = storage.size(relative_output_path)
-        upload_complete = chunk_index + 1 == chunk_count
-        if upload_complete and size != chunk_total_size:
-            raise HttpError(
-                409,
-                f"ArchiveResult chunk size mismatch for {relative_output_path}: expected {chunk_total_size}, got {size}",
-            )
-
-        guessed_mime = mimetypes.guess_type(relative_output_path)[0]
-        output_mime_type = (mime_types[0] if mime_types else "") or uploaded_file.content_type or guessed_mime or "application/octet-stream"
-        output_files[relative_output_path] = {
-            "extension": PurePosixPath(relative_output_path).suffix.lower().lstrip("."),
-            "mimetype": output_mime_type,
-            "size": size,
-            "upload": {
-                "chunked": True,
-                "chunk_index": chunk_index,
-                "chunk_count": chunk_count,
-                "chunks_received": chunk_index + 1,
-                "complete": upload_complete,
-            },
-        }
-
-        return output_files
-
-    for index, uploaded_file in enumerate(files):
-        relative_output_path = _normalize_uploaded_archiveresult_output_path(
-            output_paths[index] if index < len(output_paths) else "",
-            filename=uploaded_file.name,
-        )
-        if storage.exists(relative_output_path):
-            storage.delete(relative_output_path)
-        saved_output_path = storage.save(relative_output_path, uploaded_file)
-        size = storage.size(saved_output_path)
-        guessed_mime = mimetypes.guess_type(saved_output_path)[0]
-        output_mime_type = (
-            (mime_types[index] if index < len(mime_types) else "")
-            or uploaded_file.content_type
-            or guessed_mime
-            or "application/octet-stream"
-        )
-        output_files[saved_output_path] = {
-            "extension": PurePosixPath(saved_output_path).suffix.lower().lstrip("."),
-            "mimetype": output_mime_type,
-            "size": size,
-        }
-
-    return output_files
-
-
-@router.post(
-    "/archiveresults",
-    response=ArchiveResultSchema,
-    url_name="create_archiveresult",
-)
-def create_archiveresult(
-    request: HttpRequest,
-    snapshot_id: str = Form(...),
-    plugin: str = Form(...),
-    output_str: str = Form(""),
-    hook_name: str = Form(ARCHIVERESULT_UPLOAD_HOOK_NAME),
-    status: str = Form(str(ArchiveResult.StatusChoices.SUCCEEDED)),
-    output_json: str = Form(""),
-):
-    """Create or update an ArchiveResult with one or more output files."""
-    snapshot = _get_snapshot_by_ref(snapshot_id)
-    plugin_name = _normalize_uploaded_archiveresult_plugin(plugin)
-    result = ArchiveResult.objects.filter(
-        snapshot=snapshot,
-        plugin=plugin_name,
-        hook_name=hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME,
-    ).first()
-    result = result or ArchiveResult(snapshot=snapshot, plugin=plugin_name, hook_name=hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME)
-    metadata = {"output_str": output_str, "status": status, "output_json": _parse_archiveresult_output_json(output_json)}
-    files = _write_archiveresult_files(request, snapshot, plugin_name, allow_empty=True)
-    try:
-        return result.apply_upload(files, metadata=metadata, replace_metadata=True)
-    except ArchiveResult.UploadConflict as err:
-        raise HttpError(409, str(err)) from err
-
-
-@router.patch("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="patch_archiveresult")
-def patch_archiveresult(
-    request: HttpRequest,
-    archiveresult_id: str,
-):
-    """Append or replace files on an existing ArchiveResult."""
-    result = ArchiveResult.objects.select_related("snapshot__crawl__created_by").get(_uuid_ref_query("id", archiveresult_id))
-    uploaded_output_files = _write_archiveresult_files(
-        request,
-        result.snapshot,
-        result.plugin,
-    )
-    output_str = _get_archiveresult_upload_form_value(request, "output_str")
-    status = _get_archiveresult_upload_form_value(request, "status")
-    output_json = _get_archiveresult_upload_form_value(request, "output_json")
-    metadata = {key: value for key, value in {"output_str": output_str, "status": status}.items() if value}
-    if output_json:
-        metadata["output_json"] = _parse_archiveresult_output_json(output_json)
-    try:
-        return result.apply_upload(uploaded_output_files, metadata=metadata)
-    except ArchiveResult.UploadConflict as err:
-        raise HttpError(409, str(err)) from err

archivebox/api/v1_core/lookup.py

@@ -1,12 +0,0 @@
-from django.db.models import Q
-
-from archivebox.core.models import Snapshot
-from archivebox.misc.db import uuid_ref_query as _uuid_ref_query
-
-
-def _get_snapshot_by_ref(snapshot_id: str):
-    queryset = Snapshot.objects.select_related("crawl__created_by")
-    try:
-        return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        return queryset.get(_uuid_ref_query("id", snapshot_id))

archivebox/api/v1_core/pagination.py

@@ -1,40 +0,0 @@
-import math
-from typing import Any
-
-from django.http import HttpRequest
-from ninja.pagination import PaginationBase
-
-
-class CustomPagination(PaginationBase):
-    class Input(PaginationBase.Input):
-        limit: int = 200
-        offset: int = 0
-        page: int = 0
-
-    class Output(PaginationBase.Output):
-        count: int
-        total_items: int
-        total_pages: int
-        page: int
-        limit: int
-        offset: int
-        num_items: int
-        items: list[Any]
-
-    def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
-        limit = min(pagination.limit, 500)
-        offset = pagination.offset or (pagination.page * limit)
-        total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count()
-        total_pages = math.ceil(total / limit)
-        current_page = math.ceil(offset / (limit + 1))
-        items = queryset[offset : offset + limit]
-        return {
-            "count": total,
-            "total_items": total,
-            "total_pages": total_pages,
-            "page": current_page,
-            "limit": limit,
-            "offset": offset,
-            "num_items": len(items),
-            "items": items,
-        }

archivebox/api/v1_core/schemas.py

@@ -1,301 +0,0 @@
-from collections import defaultdict
-from datetime import datetime
-from typing import Annotated, Any
-from uuid import UUID
-
-from django.db.models import Q
-from archivebox.api.schemas import OwnedObjectSchema
-
-from ninja import FilterLookup, FilterSchema, Schema
-
-from archivebox.core.models import ArchiveResult, Snapshot
-
-
-class MinimalArchiveResultSchema(OwnedObjectSchema):
-    TYPE: str = "core.models.ArchiveResult"
-    id: UUID
-    created_at: datetime | None
-    modified_at: datetime | None
-    status: str
-    retry_at: datetime | None = None
-    plugin: str
-    hook_name: str
-    process_id: UUID | None
-    cmd_version: str | None
-    cmd: list[str] | None
-    pwd: str | None
-    output_str: str
-    output_json: dict[str, Any] | None
-    output_files: dict[str, dict[str, Any]] | None
-    output_size: int
-    output_mimetypes: str
-    start_ts: datetime | None
-    end_ts: datetime | None
-
-    @staticmethod
-    def resolve_output_files(obj):
-        return obj.output_file_map()
-
-    @staticmethod
-    def resolve_output_mimetypes(obj) -> str:
-        mime_sizes: dict[str, int] = defaultdict(int)
-        for metadata in obj.output_file_map().values():
-            if not isinstance(metadata, dict):
-                continue
-            mimetype = str(metadata.get("mimetype") or "").strip()
-            try:
-                size = max(int(metadata.get("size") or 0), 0)
-            except (TypeError, ValueError):
-                size = 0
-            if mimetype and size:
-                mime_sizes[mimetype] += size
-        if mime_sizes:
-            return ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
-        return obj.output_mimetypes or ""
-
-
-class ArchiveResultSchema(MinimalArchiveResultSchema):
-    TYPE: str = "core.models.ArchiveResult"
-    snapshot_id: UUID
-    snapshot_timestamp: str
-    snapshot_url: str
-    snapshot_tags: list[str]
-
-    @staticmethod
-    def resolve_snapshot_timestamp(obj):
-        return obj.snapshot.timestamp
-
-    @staticmethod
-    def resolve_snapshot_url(obj):
-        return obj.snapshot.url
-
-    @staticmethod
-    def resolve_snapshot_id(obj):
-        return obj.snapshot_id
-
-    @staticmethod
-    def resolve_snapshot_tags(obj):
-        return sorted(tag.name for tag in obj.snapshot.tags.all())
-
-
-class ArchiveResultFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup(["id__startswith", "snapshot__id__startswith", "snapshot__timestamp__startswith"])] = None
-    search: Annotated[
-        str | None,
-        FilterLookup(
-            [
-                "snapshot__url__icontains",
-                "snapshot__title__icontains",
-                "snapshot__tags__name__icontains",
-                "plugin",
-                "output_str__icontains",
-                "id__startswith",
-                "snapshot__id__startswith",
-                "snapshot__timestamp__startswith",
-            ],
-        ),
-    ] = None
-    snapshot_id: Annotated[str | None, FilterLookup(["snapshot__id__startswith", "snapshot__timestamp__startswith"])] = None
-    snapshot_url: Annotated[str | None, FilterLookup("snapshot__url__icontains")] = None
-    snapshot_tag: Annotated[str | None, FilterLookup("snapshot__tags__name__icontains")] = None
-    status: Annotated[str | None, FilterLookup("status")] = None
-    output_str: Annotated[str | None, FilterLookup("output_str__icontains")] = None
-    plugin: Annotated[str | None, FilterLookup("plugin__icontains")] = None
-    hook_name: Annotated[str | None, FilterLookup("hook_name__icontains")] = None
-    process_id: Annotated[str | None, FilterLookup("process__id__startswith")] = None
-    cmd: Annotated[str | None, FilterLookup("cmd__0__icontains")] = None
-    pwd: Annotated[str | None, FilterLookup("pwd__icontains")] = None
-    cmd_version: Annotated[str | None, FilterLookup("cmd_version")] = None
-    created_at: Annotated[datetime | None, FilterLookup("created_at")] = None
-    created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
-    created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
-
-
-class SnapshotSchema(OwnedObjectSchema):
-    TYPE: str = "core.models.Snapshot"
-    id: UUID
-    created_at: datetime
-    modified_at: datetime
-    status: str
-    retry_at: datetime | None
-    bookmarked_at: datetime
-    downloaded_at: datetime | None
-    url: str
-    tags: list[str]
-    title: str | None
-    timestamp: str
-    archive_path: str
-    archive_size: int
-    output_size: int
-    num_archiveresults: int
-    archiveresults: list[MinimalArchiveResultSchema]
-
-    @staticmethod
-    def resolve_tags(obj):
-        return sorted(tag.name for tag in obj.tags.all())
-
-    @staticmethod
-    def resolve_archive_size(obj):
-        return int(obj.archive_size or 0)
-
-    @staticmethod
-    def resolve_output_size(obj):
-        return SnapshotSchema.resolve_archive_size(obj)
-
-    @staticmethod
-    def resolve_num_archiveresults(obj, context):
-        return obj.archiveresult_set.all().distinct().count()
-
-    @staticmethod
-    def resolve_archiveresults(obj, context):
-        if bool(context["request"].__dict__.get("with_archiveresults", False)):
-            return obj.archiveresult_set.all().distinct()
-        return ArchiveResult.objects.none()
-
-
-class SnapshotUpdateSchema(Schema):
-    action: str | None = None
-    status: str | None = None
-    retry_at: datetime | None = None
-    tags: list[str] | None = None
-
-
-class SnapshotCreateSchema(Schema):
-    url: str
-    crawl_id: str | None = None
-    depth: int = 0
-    title: str | None = None
-    tags: list[str] | None = None
-    status: str | None = None
-
-
-class SnapshotDeleteResponseSchema(Schema):
-    success: bool
-    snapshot_id: str
-    crawl_id: str
-    deleted_count: int
-
-
-class SnapshotFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup(["id__istartswith", "id__iendswith", "timestamp__startswith"])] = None
-    created_by_id: Annotated[str | None, FilterLookup("crawl__created_by_id")] = None
-    created_by_username: Annotated[str | None, FilterLookup("crawl__created_by__username__icontains")] = None
-    created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
-    created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
-    created_at: Annotated[datetime | None, FilterLookup("created_at")] = None
-    modified_at: Annotated[datetime | None, FilterLookup("modified_at")] = None
-    modified_at__gte: Annotated[datetime | None, FilterLookup("modified_at__gte")] = None
-    modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
-    search: str | None = None
-    search_mode: str | None = None
-    status: str | None = None
-    url: Annotated[str | None, FilterLookup("url")] = None
-    tag: Annotated[str | None, FilterLookup("tags__name")] = None
-    title: Annotated[str | None, FilterLookup("title__icontains")] = None
-    timestamp: Annotated[str | None, FilterLookup("timestamp__startswith")] = None
-    bookmarked_at__gte: Annotated[datetime | None, FilterLookup("bookmarked_at__gte")] = None
-    bookmarked_at__lt: Annotated[datetime | None, FilterLookup("bookmarked_at__lt")] = None
-
-    def filter_search(self, value: str | None) -> Q:
-        return Q()
-
-    def filter_search_mode(self, value: str | None) -> Q:
-        return Q()
-
-    def filter_status(self, value: str | None) -> Q:
-        return Q()
-
-
-class TagSchema(OwnedObjectSchema):
-    TYPE: str = "core.models.Tag"
-    id: int
-    modified_at: datetime
-    created_at: datetime
-    name: str
-    num_snapshots: int
-    snapshots: list[SnapshotSchema]
-
-    @staticmethod
-    def resolve_num_snapshots(obj, context):
-        return obj.snapshot_set.all().distinct().count()
-
-    @staticmethod
-    def resolve_snapshots(obj, context):
-        if bool(context["request"].__dict__.get("with_snapshots", False)):
-            return obj.snapshot_set.all().distinct()
-        return Snapshot.objects.none()
-
-
-class TagAutocompleteSchema(Schema):
-    tags: list[dict]
-
-
-class TagCreateSchema(Schema):
-    name: str
-
-
-class TagCreateResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    tag_name: str
-    created: bool
-
-
-class TagSearchSnapshotSchema(Schema):
-    id: str
-    title: str
-    url: str
-    favicon_url: str
-    admin_url: str
-    archive_url: str
-    downloaded_at: str | None = None
-
-
-class TagSearchCardSchema(Schema):
-    id: int
-    name: str
-    slug: str
-    num_snapshots: int
-    filter_url: str
-    edit_url: str
-    export_urls_url: str
-    export_jsonl_url: str
-    rename_url: str
-    delete_url: str
-    snapshots: list[TagSearchSnapshotSchema]
-
-
-class TagSearchResponseSchema(Schema):
-    tags: list[TagSearchCardSchema]
-    sort: str
-    created_by: str
-    year: str
-    has_snapshots: str
-
-
-class TagUpdateSchema(Schema):
-    name: str
-
-
-class TagUpdateResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    tag_name: str
-
-
-class TagDeleteResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    deleted_count: int
-
-
-class TagSnapshotRequestSchema(Schema):
-    snapshot_id: str
-    tag_name: str | None = None
-    tag_id: int | None = None
-
-
-class TagSnapshotResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    tag_name: str

archivebox/api/v1_core/snapshots.py

@@ -1,356 +0,0 @@
-from datetime import datetime, time
-from html import unescape
-from typing import Any
-
-from django.contrib.auth import get_user_model
-from django.contrib.auth.models import User
-from django.core.exceptions import ValidationError
-from django.db import IntegrityError
-from django.db.models import Q
-from django.http import HttpRequest, HttpResponse
-from django.utils import timezone
-from django.utils.dateparse import parse_date, parse_datetime
-from django.utils.feedgenerator import Rss201rev2Feed
-from ninja import Query, Router
-from ninja.errors import HttpError
-from ninja.pagination import paginate
-
-from archivebox.api.v1_crawls import get_crawl_by_ref
-from archivebox.core.models import Snapshot
-from archivebox.core.routes_util import build_web_url
-from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status
-from archivebox.crawls.locks import crawl_lifecycle_lock
-from archivebox.crawls.models import Crawl
-from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
-from archivebox.search.config import get_search_mode, get_search_mode_backend
-from archivebox.search.query import apply_snapshot_search
-
-from .lookup import _get_snapshot_by_ref
-from .pagination import CustomPagination
-from .schemas import SnapshotCreateSchema, SnapshotDeleteResponseSchema, SnapshotFilterSchema, SnapshotSchema, SnapshotUpdateSchema
-
-router = Router(tags=["Core Models"])
-
-
-def normalize_tag_list(tags: list[str] | None = None) -> list[str]:
-    return [tag.strip() for tag in (tags or []) if tag and tag.strip()]
-
-
-def _parse_rss_before(before: str | None) -> datetime:
-    if not before:
-        return timezone.now()
-
-    value = before.strip()
-    parsed_dt = None
-
-    if len(value) == 8 and value.isdigit():
-        parsed_date = datetime.strptime(value, "%Y%m%d").date()
-    else:
-        parsed_dt = parse_datetime(value)
-        parsed_date = None if parsed_dt else parse_date(value)
-
-    if parsed_dt is None:
-        if parsed_date is None:
-            raise HttpError(400, "before must be an ISO datetime, YYYY-MM-DD, or YYYYMMDD")
-        parsed_dt = datetime.combine(parsed_date, time.max)
-
-    if timezone.is_naive(parsed_dt):
-        parsed_dt = timezone.make_aware(parsed_dt, timezone.get_current_timezone())
-    return parsed_dt
-
-
-def _filter_snapshots_for_rss(
-    *,
-    crawl_id: str = "",
-    created_by: str = "",
-    before: str | None = None,
-    limit: int = 50,
-):
-    limit = max(1, min(int(limit or 50), 500))
-    before_dt = _parse_rss_before(before)
-    queryset = (
-        Snapshot.objects.select_related("crawl__created_by")
-        .prefetch_related("tags")
-        .only(
-            "id",
-            "url",
-            "title",
-            "timestamp",
-            "bookmarked_at",
-            "created_at",
-            "modified_at",
-            "fs_version",
-            "crawl_id",
-            "crawl__id",
-            "crawl__created_by_id",
-            "crawl__created_by__id",
-            "crawl__created_by__username",
-        )
-        .filter(bookmarked_at__lte=before_dt)
-    )
-    crawl_id = crawl_id.strip()
-    if crawl_id:
-        matching_crawl_pks = list(filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).values_list("pk", flat=True)[:100])
-        queryset = queryset.filter(crawl_id__in=matching_crawl_pks)
-
-    created_by = created_by.strip()
-    if created_by:
-        created_by_query = Q(crawl__created_by__username__iexact=created_by)
-        user_model = get_user_model()
-        try:
-            prepared_pk = user_model._meta.pk.get_prep_value(created_by)
-        except (TypeError, ValueError, ValidationError):
-            prepared_pk = None
-        if prepared_pk not in (None, ""):
-            created_by_query |= Q(crawl__created_by_id=prepared_pk)
-        queryset = queryset.filter(created_by_query)
-
-    return queryset.order_by("-bookmarked_at", "-created_at", "-id")[:limit]
-
-
-def _snapshots_rss_response(
-    request: HttpRequest,
-    *,
-    snapshots,
-    title: str = "ArchiveBox Snapshots",
-) -> HttpResponse:
-    web_base_url = build_web_url("/", request=request).rstrip("/")
-    feed_query = request.GET.copy()
-    for sensitive_param in ("api_key", "token", "password"):
-        feed_query.pop(sensitive_param, None)
-    feed_path = request.path
-    feed_url = request.build_absolute_uri(f"{feed_path}?{feed_query.urlencode()}" if feed_query else feed_path)
-
-    feed = Rss201rev2Feed(
-        title=title,
-        link=build_web_url("/public/", request=request),
-        description="Recently added ArchiveBox snapshots.",
-        language="en",
-        feed_url=feed_url,
-    )
-
-    for snapshot in snapshots:
-        archived_url = build_web_url(f"/{snapshot.archive_path_from_db}", request=request)
-        tags = [tag.name for tag in snapshot.tags.all()]
-        crawl_user = snapshot.crawl.created_by if snapshot.crawl_id else None
-        description = f"Original URL: {snapshot.url}\nArchived snapshot: {archived_url}"
-        feed.add_item(
-            title=unescape(snapshot.title or snapshot.url),
-            link=archived_url or web_base_url,
-            description=description,
-            unique_id=str(snapshot.id),
-            unique_id_is_permalink=False,
-            pubdate=snapshot.bookmarked_at or snapshot.created_at,
-            updateddate=snapshot.modified_at,
-            author_name=crawl_user.username if crawl_user else None,
-            categories=tags,
-        )
-
-    return HttpResponse(feed.writeString("utf-8"), content_type="application/rss+xml; charset=utf-8")
-
-
-@router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
-@paginate(CustomPagination)
-def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
-    """List all Snapshot entries matching these filters."""
-    setattr(request, "with_archiveresults", with_archiveresults)
-    try:
-        queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-    queryset = filters.filter(queryset).distinct()
-    query = (filters.search or "").strip()
-    if not query:
-        return queryset
-
-    runtime_config = request.archivebox_config
-    search_mode = get_search_mode(filters.search_mode, config=runtime_config)
-    try:
-        return apply_snapshot_search(
-            queryset,
-            query,
-            search_mode=search_mode,
-            config=runtime_config,
-            include_id_matches=True,
-        )
-    except Exception:
-        if get_search_mode_backend(search_mode, config=runtime_config):
-            return queryset.none()
-        return apply_snapshot_search(queryset, query, search_mode="meta", config=runtime_config, include_id_matches=True)
-
-
-@router.get("/snapshots.rss", url_name="get_snapshots_rss")
-def get_snapshots_rss(
-    request: HttpRequest,
-    crawl_id: str = "",
-    created_by: str = "",
-    limit: int = 50,
-    before: str | None = None,
-):
-    """Return matching snapshots as an RSS feed, newest first."""
-    snapshots = _filter_snapshots_for_rss(
-        crawl_id=crawl_id,
-        created_by=created_by,
-        limit=limit,
-        before=before,
-    )
-    return _snapshots_rss_response(request, snapshots=snapshots)
-
-
-@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
-def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
-    """Get a specific Snapshot by id."""
-    setattr(request, "with_archiveresults", with_archiveresults)
-    return _get_snapshot_by_ref(snapshot_id)
-
-
-@router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
-def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
-    tags = normalize_tag_list(data.tags)
-    try:
-        status = normalize_snapshot_status(data.status)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-    if not data.url.strip():
-        raise HttpError(400, "URL is required")
-    try:
-        validate_url_length(data.url.strip())
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-    if data.depth not in (0, 1, 2, 3, 4):
-        raise HttpError(400, "depth must be between 0 and 4")
-
-    if data.crawl_id:
-        crawl = get_crawl_by_ref(data.crawl_id)
-        crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
-        tags = tags or crawl_tags
-    else:
-        crawl = Crawl.objects.create(
-            urls=data.url,
-            max_depth=max(data.depth, 0),
-            tags_str=",".join(tags),
-            status=Crawl.StatusChoices.QUEUED,
-            retry_at=timezone.now(),
-            created_by=request.user if isinstance(request.user, User) else None,
-        )
-
-    # Browser uploads must not wait behind the runner's crawl-wide lifecycle
-    # lock. The unique insert recovery and CAS update below are the request-side
-    # coordination boundary for this idempotent metadata sync.
-    snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
-    if snapshot is None:
-        try:
-            snapshot = Snapshot.objects.create(
-                url=data.url,
-                crawl=crawl,
-                depth=data.depth,
-                title=data.title,
-                timestamp=str(timezone.now().timestamp()),
-                status=status or Snapshot.StatusChoices.QUEUED,
-                retry_at=timezone.now(),
-            )
-        except IntegrityError:
-            snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
-            if snapshot is None:
-                raise
-
-    for _attempt in range(3):
-        updates: dict[str, Any] = {}
-        if data.title is not None and snapshot.title != data.title:
-            updates["title"] = data.title
-        if status is not None and snapshot.status != status:
-            updates["status"] = status
-        if not updates or snapshot.safe_update(updates, extra_filter={"modified_at": snapshot.modified_at}):
-            break
-    else:
-        raise HttpError(409, "Snapshot changed while metadata was being updated")
-
-    if tags:
-        snapshot.save_tags(
-            tags,
-            created_by=request.user if isinstance(request.user, User) else None,
-        )
-
-    try:
-        snapshot.ensure_crawl_symlink()
-    except Exception:
-        pass
-
-    setattr(request, "with_archiveresults", False)
-    return snapshot
-
-
-@router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
-def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
-    """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
-    snapshot = _get_snapshot_by_ref(snapshot_id)
-    crawl_id = str(snapshot.crawl_id)
-
-    payload = data.dict(exclude_unset=True)
-    update_fields = ["modified_at"]
-    action = payload.pop("action", None)
-    tags = payload.pop("tags", None)
-
-    if action:
-        if action == "pause":
-            snapshot.pause()
-            setattr(request, "with_archiveresults", False)
-            return snapshot
-        if action in ("resume", "unpause"):
-            snapshot.resume()
-            setattr(request, "with_archiveresults", False)
-            return snapshot
-        if action == "cancel":
-            snapshot.cancel()
-            setattr(request, "with_archiveresults", False)
-            return snapshot
-        raise HttpError(400, f"Invalid action: {action}")
-
-    with crawl_lifecycle_lock(crawl_id):
-        snapshot = _get_snapshot_by_ref(snapshot_id)
-        if "status" in payload:
-            try:
-                snapshot.status = normalize_snapshot_status(payload["status"])
-            except ValueError as err:
-                raise HttpError(400, str(err)) from err
-            if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
-                snapshot.retry_at = None
-            update_fields.append("status")
-
-        if "retry_at" in payload:
-            snapshot.retry_at = payload["retry_at"]
-            update_fields.append("retry_at")
-
-        if tags is not None:
-            snapshot.save_tags(
-                normalize_tag_list(tags),
-                created_by=request.user if isinstance(request.user, User) else None,
-            )
-
-        if payload.get("status") == Snapshot.StatusChoices.SEALED:
-            snapshot.cancel()
-        else:
-            snapshot.save(update_fields=update_fields)
-    setattr(request, "with_archiveresults", False)
-    return snapshot
-
-
-@router.delete("/snapshot/{snapshot_id}", response=SnapshotDeleteResponseSchema, url_name="delete_snapshot")
-def delete_snapshot(request: HttpRequest, snapshot_id: str):
-    snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
-    snapshot_id_str = str(snapshot.id)
-    crawl_id_str = str(snapshot.crawl.pk)
-    snapshot.cancel()
-
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        snapshot = get_snapshot(request, snapshot_id_str, with_archiveresults=False)
-        deleted_count, _ = snapshot.delete()
-    return {
-        "success": True,
-        "snapshot_id": snapshot_id_str,
-        "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
-    }

archivebox/api/v1_core/tags.py

@@ -1,285 +0,0 @@
-from django.core.exceptions import ValidationError
-from django.db.models import Q
-from django.http import HttpRequest, HttpResponse
-from ninja import Router
-from ninja.errors import HttpError
-from ninja.pagination import paginate
-
-from archivebox.api.auth import authenticated_user_from_request
-from archivebox.config.common import get_config
-from archivebox.core.models import Snapshot, Tag
-from archivebox.core.permissions import public_snapshots_queryset
-from archivebox.core.tag_util import (
-    add_snapshot_counts,
-    build_tag_cards,
-    export_tag_snapshots_jsonl,
-    export_tag_urls,
-    get_matching_tags,
-    get_tag_by_ref,
-    normalize_created_by_filter,
-    normalize_created_year_filter,
-    normalize_has_snapshots_filter,
-    normalize_tag_sort,
-)
-
-from .pagination import CustomPagination
-from .schemas import (
-    TagAutocompleteSchema,
-    TagCreateResponseSchema,
-    TagCreateSchema,
-    TagDeleteResponseSchema,
-    TagSchema,
-    TagSearchResponseSchema,
-    TagSnapshotRequestSchema,
-    TagSnapshotResponseSchema,
-    TagUpdateResponseSchema,
-    TagUpdateSchema,
-)
-
-router = Router(tags=["Core Models"])
-
-
-@router.get("/tags", response=list[TagSchema], url_name="get_tags")
-@paginate(CustomPagination)
-def get_tags(request: HttpRequest):
-    setattr(request, "with_snapshots", False)
-    setattr(request, "with_archiveresults", False)
-    return get_matching_tags()
-
-
-@router.get("/tag/{tag_id}", response=TagSchema, url_name="get_tag")
-def get_tag(request: HttpRequest, tag_id: str, with_snapshots: bool = True):
-    setattr(request, "with_snapshots", with_snapshots)
-    setattr(request, "with_archiveresults", False)
-    try:
-        return get_tag_by_ref(tag_id)
-    except (Tag.DoesNotExist, ValidationError):
-        raise HttpError(404, "Tag not found")
-
-
-def _get_snapshot_for_tag_edit(snapshot_ref: str) -> Snapshot:
-    snapshot_ref = str(snapshot_ref or "").strip().lower()
-    if not snapshot_ref:
-        raise HttpError(400, "Snapshot id is required")
-
-    snapshot_qs = Snapshot.objects.only("id")
-    is_full_uuid = len(snapshot_ref.replace("-", "")) == 32 and all(char in "0123456789abcdef-" for char in snapshot_ref)
-    if is_full_uuid:
-        try:
-            return snapshot_qs.get(pk=snapshot_ref.replace("-", ""))
-        except (Snapshot.DoesNotExist, ValueError):
-            pass
-
-    if len(snapshot_ref) >= 14:
-        try:
-            return snapshot_qs.get(timestamp=snapshot_ref)
-        except Snapshot.DoesNotExist:
-            pass
-        except Snapshot.MultipleObjectsReturned:
-            snapshot = snapshot_qs.filter(timestamp=snapshot_ref).first()
-            if snapshot is not None:
-                return snapshot
-
-    try:
-        return snapshot_qs.get(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref))
-    except Snapshot.DoesNotExist:
-        raise HttpError(404, "Snapshot not found") from None
-    except Snapshot.MultipleObjectsReturned:
-        snapshot = snapshot_qs.filter(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref)).first()
-        if snapshot is None:
-            raise HttpError(404, "Snapshot not found")
-        return snapshot
-
-
-@router.get("/tags/search/", response=TagSearchResponseSchema, url_name="search_tags")
-def search_tags(
-    request: HttpRequest,
-    q: str = "",
-    sort: str = "created_desc",
-    created_by: str = "",
-    year: str = "",
-    has_snapshots: str = "all",
-):
-    """Return detailed tag cards for admin/live-search UIs."""
-    normalized_sort = normalize_tag_sort(sort)
-    normalized_created_by = normalize_created_by_filter(created_by)
-    normalized_year = normalize_created_year_filter(year)
-    normalized_has_snapshots = normalize_has_snapshots_filter(has_snapshots)
-    return {
-        "tags": build_tag_cards(
-            query=q,
-            request=request,
-            preview_limit=0,
-            sort=normalized_sort,
-            created_by=normalized_created_by,
-            year=normalized_year,
-            has_snapshots=normalized_has_snapshots,
-        ),
-        "sort": normalized_sort,
-        "created_by": normalized_created_by,
-        "year": normalized_year,
-        "has_snapshots": normalized_has_snapshots,
-    }
-
-
-def _public_tag_listing_enabled() -> bool:
-    return get_config().PUBLIC_INDEX
-
-
-def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
-    if authenticated_user_from_request(request):
-        return True
-
-    return _public_tag_listing_enabled()
-
-
-@router.get("/tags/autocomplete/", response=TagAutocompleteSchema, url_name="tags_autocomplete", auth=None)
-def tags_autocomplete(request: HttpRequest, q: str = ""):
-    """Return tags matching the query for autocomplete."""
-    if not _request_has_tag_autocomplete_access(request):
-        raise HttpError(401, "Authentication required")
-
-    public_only = not request.user.is_authenticated and not request.__dict__.get("_api_token")
-    queryset = get_matching_tags(q)
-    public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
-    if public_only:
-        queryset = queryset.filter(snapshot_set__id__in=public_snapshots.values("id")).distinct()
-    tags = list(queryset[: 50 if not q else 20])
-    add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)
-
-    return {
-        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": tag.__dict__.get("num_snapshots", 0)} for tag in tags],
-    }
-
-
-@router.post("/tags/create/", response=TagCreateResponseSchema, url_name="tags_create")
-def tags_create(request: HttpRequest, data: TagCreateSchema):
-    """Create a new tag or return existing one."""
-    try:
-        tag, created = Tag.get_or_create_by_name(
-            data.name,
-            created_by=request.user if request.user.is_authenticated else None,
-        )
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-        "created": created,
-    }
-
-
-@router.post("/tag/{tag_id}/rename", response=TagUpdateResponseSchema, url_name="rename_tag")
-def rename_tag(request: HttpRequest, tag_id: int, data: TagUpdateSchema):
-    try:
-        tag = get_tag_by_ref(tag_id).rename(data.name)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-    }
-
-
-@router.delete("/tag/{tag_id}", response=TagDeleteResponseSchema, url_name="delete_tag")
-def delete_tag(request: HttpRequest, tag_id: int):
-    try:
-        tag = get_tag_by_ref(tag_id)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-
-    deleted_count, _ = tag.delete()
-    return {
-        "success": True,
-        "tag_id": int(tag_id),
-        "deleted_count": deleted_count,
-    }
-
-
-@router.get("/tag/{tag_id}/urls.txt", url_name="tag_urls_export")
-def tag_urls_export(request: HttpRequest, tag_id: int):
-    try:
-        tag = get_tag_by_ref(tag_id)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-
-    response = HttpResponse(export_tag_urls(tag), content_type="text/plain; charset=utf-8")
-    response["Content-Disposition"] = f'attachment; filename="tag-{tag.slug}-urls.txt"'
-    return response
-
-
-@router.get("/tag/{tag_id}/snapshots.jsonl", url_name="tag_snapshots_export")
-def tag_snapshots_export(request: HttpRequest, tag_id: int):
-    try:
-        tag = get_tag_by_ref(tag_id)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-
-    response = HttpResponse(export_tag_snapshots_jsonl(tag), content_type="application/x-ndjson; charset=utf-8")
-    response["Content-Disposition"] = f'attachment; filename="tag-{tag.slug}-snapshots.jsonl"'
-    return response
-
-
-@router.post("/tags/add-to-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_add_to_snapshot")
-def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
-    """Add a tag to a snapshot. Creates the tag if it doesn't exist."""
-    snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
-
-    # Get or create the tag
-    if data.tag_name:
-        try:
-            tag, _ = Tag.get_or_create_by_name(
-                data.tag_name,
-                created_by=request.user if request.user.is_authenticated else None,
-            )
-        except ValueError as err:
-            raise HttpError(400, str(err)) from err
-    elif data.tag_id:
-        try:
-            tag = get_tag_by_ref(data.tag_id)
-        except Tag.DoesNotExist:
-            raise HttpError(404, "Tag not found")
-    else:
-        raise HttpError(400, "Either tag_name or tag_id is required")
-
-    snapshot.add_tag_ids([tag.pk])
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-    }
-
-
-@router.post("/tags/remove-from-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_remove_from_snapshot")
-def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
-    """Remove a tag from a snapshot."""
-    snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
-
-    # Get the tag
-    if data.tag_id:
-        try:
-            tag = Tag.objects.get(pk=data.tag_id)
-        except Tag.DoesNotExist:
-            raise HttpError(404, "Tag not found")
-    elif data.tag_name:
-        try:
-            tag = Tag.objects.get(name__iexact=data.tag_name.strip())
-        except Tag.DoesNotExist:
-            raise HttpError(404, "Tag not found")
-    else:
-        raise HttpError(400, "Either tag_name or tag_id is required")
-
-    snapshot.remove_tag_ids([tag.pk])
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-    }

archivebox/api/v1_crawls.py

@@ -7,10 +7,9 @@ from django.http import FileResponse, HttpRequest
 from django.shortcuts import redirect
 from django.utils import timezone
 
+from django.contrib.auth import get_user_model
 from django.contrib.auth.models import User
 
-from archivebox.api.schemas import OwnedObjectSchema
-
 from ninja import Router, Schema
 from ninja.errors import HttpError
 
@@ -30,13 +29,15 @@ from .auth import API_AUTH_METHODS, authenticated_user_from_request
 router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
 
 
-class CrawlSchema(OwnedObjectSchema):
+class CrawlSchema(Schema):
     TYPE: str = "crawls.models.Crawl"
 
     id: UUID
 
     modified_at: datetime
     created_at: datetime
+    created_by_id: str
+    created_by_username: str
 
     status: str
     retry_at: datetime | None
@@ -47,6 +48,17 @@ class CrawlSchema(OwnedObjectSchema):
     tags_str: str
     config: dict
 
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by_id)
+
+    @staticmethod
+    def resolve_created_by_username(obj):
+        user_model = get_user_model()
+        user = user_model.objects.get(id=obj.created_by_id)
+        username = user.username
+        return username if isinstance(username, str) else str(user)
+
     @staticmethod
     def resolve_config(obj):
         # Redact credential values so REST responses can never leak the raw

archivebox/api/v1_personas.py

@@ -7,8 +7,6 @@ from uuid import UUID
 
 from django.db.models import Q
 from django.http import HttpRequest
-from archivebox.api.schemas import OwnedObjectSchema
-
 from ninja import Router, Schema
 from ninja.pagination import paginate
 from pydantic import Field
@@ -38,13 +36,23 @@ class PersonaSyncSchema(Schema):
     auth_json: dict[str, Any] = Field(default_factory=dict)
 
 
-class PersonaSchema(OwnedObjectSchema):
+class PersonaSchema(Schema):
     TYPE: str = "personas.models.Persona"
     id: UUID
     name: str
     created_at: datetime
+    created_by_id: str
+    created_by_username: str
     config: dict[str, Any] | None
 
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by.pk)
+
+    @staticmethod
+    def resolve_created_by_username(obj) -> str:
+        return obj.created_by.username
+
     @staticmethod
     def resolve_config(obj):
         # Redact credential values so REST responses don't leak the raw

archivebox/base_models/admin.py

@@ -13,16 +13,10 @@ from django.db import DatabaseError, models
 from django.forms.renderers import BaseRenderer
 from django.http import HttpRequest, QueryDict
 from django.urls import path, register_converter
-from django.utils.html import format_html
 from django.utils.safestring import SafeString, mark_safe
 from django_object_actions import DjangoObjectActions
 
 
-def card_fieldset(title: str | None, fields: tuple, *, wide: bool = False, **options):
-    """Build a Django fieldset using the shared card layout."""
-    return title, {"fields": fields, "classes": ("card", "wide") if wide else ("card",), **options}
-
-
 class HexUUIDConverter:
     """URL path converter that canonicalizes UUIDs to their 32-char hex form.
 
@@ -87,11 +81,6 @@ class KeyValueWidget(forms.Widget):
             options: dict[str, ConfigOption] = {}
             for key, metadata in config_field_metadata().items():
                 option_type = metadata.get("type", "string")
-                # Core fields expose Python type names; plugins use JSON Schema.
-                # The editor should apply the same typed validation to both.
-                type_names = {"bool": "boolean", "int": "integer", "float": "number", "str": "string", "dict": "object", "list": "array"}
-                if isinstance(option_type, str):
-                    option_type = type_names.get(option_type, option_type)
                 option: ConfigOption = {
                     "plugin": str(metadata.get("plugin", "archivebox")),
                     "type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
@@ -129,36 +118,650 @@ class KeyValueWidget(forms.Widget):
         attrs: Mapping[str, str] | None = None,
         renderer: BaseRenderer | None = None,
     ) -> SafeString:
-        from django.template.loader import render_to_string
+        data = self._parse_value(value)
 
         widget_id = attrs.get("id", name) if attrs else name
-        options = self._get_config_options()
-        rows = [
-            mark_safe(self._render_row(widget_id, key, val if isinstance(val, str) else json.dumps(val)))
-            for key, val in self._parse_value(value).items()
-        ]
-        empty_row = mark_safe(self._render_row(widget_id, "", ""))
-        # JSON is embedded in a script, so escape HTML delimiters without changing data.
-        metadata = json.dumps(options).translate({ord("<"): r"\u003C", ord(">"): r"\u003E", ord("&"): r"\u0026"})
-        return mark_safe(
-            render_to_string(
-                "admin/widgets/config.html",
-                {
-                    "name": name,
-                    "widget_id": widget_id,
-                    "datalist_options": mark_safe(
-                        "\n".join(
-                            f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
-                            for key, opt in sorted(options.items())
-                        ),
-                    ),
-                    "config_meta_json": metadata,
-                    "rows": [*rows, empty_row],
-                    "empty_row": empty_row,
-                },
-            ),
+        config_options = self._get_config_options()
+
+        # Build datalist options
+        datalist_options = "\n".join(
+            f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
+            for key, opt in sorted(config_options.items())
         )
 
+        # Build config metadata as JSON for JS
+        config_meta_json = json.dumps(config_options)
+
+        html = f'''
+        <div id="{widget_id}_container" class="key-value-editor" style="width: 100%; max-width: none;">
+            <datalist id="{widget_id}_keys">
+                {datalist_options}
+            </datalist>
+            <div id="{widget_id}_rows" class="key-value-rows">
+        '''
+
+        # Render existing key-value pairs
+        for key, val in data.items():
+            val_str = json.dumps(val) if not isinstance(val, str) else val
+            html += self._render_row(widget_id, key, val_str)
+
+        # Always add one empty row for new entries
+        html += self._render_row(widget_id, "", "")
+
+        html += f'''
+            </div>
+            <div style="display: flex; gap: 8px; align-items: center; margin-top: 8px;">
+                <button type="button" onclick="addKeyValueRow_{widget_id}()"
+                        style="padding: 4px 12px; cursor: pointer; background: #417690; color: white; border: none; border-radius: 4px;">
+                    + Add Row
+                </button>
+            </div>
+            <input type="hidden" name="{name}" id="{widget_id}" value="">
+            <script>
+                (function() {{
+                    var configMeta_{widget_id} = {config_meta_json};
+                    var rowCounter_{widget_id} = 0;
+
+                    function stringifyValue_{widget_id}(value) {{
+                        return typeof value === 'string' ? value : JSON.stringify(value);
+                    }}
+
+                    function getTypes_{widget_id}(meta) {{
+                        if (!meta || meta.type === undefined || meta.type === null) {{
+                            return [];
+                        }}
+                        return Array.isArray(meta.type) ? meta.type : [meta.type];
+                    }}
+
+                    function getMetaForKey_{widget_id}(key) {{
+                        if (!key) {{
+                            return null;
+                        }}
+
+                        var explicitMeta = configMeta_{widget_id}[key];
+                        if (explicitMeta) {{
+                            return Object.assign({{ key: key }}, explicitMeta);
+                        }}
+
+                        if (key.endsWith('_BINARY')) {{
+                            return {{
+                                key: key,
+                                plugin: 'custom',
+                                type: 'string',
+                                default: '',
+                                description: 'Path to binary executable',
+                            }};
+                        }}
+
+                        if (isRegexConfigKey_{widget_id}(key)) {{
+                            return {{
+                                key: key,
+                                plugin: 'custom',
+                                type: 'string',
+                                default: '',
+                                description: 'Regex pattern list',
+                            }};
+                        }}
+
+                        return null;
+                    }}
+
+                    function describeMeta_{widget_id}(meta) {{
+                        if (!meta) {{
+                            return '';
+                        }}
+
+                        var details = '';
+                        if (Array.isArray(meta.enum) && meta.enum.length) {{
+                            details = 'Allowed: ' + meta.enum.map(stringifyValue_{widget_id}).join(', ');
+                        }} else {{
+                            var types = getTypes_{widget_id}(meta);
+                            if (types.length) {{
+                                details = 'Expected: ' + types.join(' or ');
+                            }}
+                        }}
+
+                        if (meta.minimum !== undefined || meta.maximum !== undefined) {{
+                            var bounds = [];
+                            if (meta.minimum !== undefined) bounds.push('min ' + meta.minimum);
+                            if (meta.maximum !== undefined) bounds.push('max ' + meta.maximum);
+                            details += (details ? ' ' : '') + '(' + bounds.join(', ') + ')';
+                        }}
+
+                        return [meta.description || '', details].filter(Boolean).join(' ');
+                    }}
+
+                    function getExampleInput_{widget_id}(key, meta) {{
+                        var types = getTypes_{widget_id}(meta);
+                        if (key.endsWith('_BINARY')) {{
+                            return 'Example: wget or /usr/bin/wget';
+                        }}
+                        if (key.endsWith('_ARGS_EXTRA') || key.endsWith('_ARGS')) {{
+                            return 'Example: ["--extra-arg"]';
+                        }}
+                        if (types.includes('array')) {{
+                            return 'Example: ["value"]';
+                        }}
+                        if (types.includes('object')) {{
+                            return 'Example: {{"key": "value"}}';
+                        }}
+                        return '';
+                    }}
+
+                    function isRegexConfigKey_{widget_id}(key) {{
+                        return key === 'URL_ALLOWLIST' ||
+                            key === 'URL_DENYLIST' ||
+                            key.endsWith('_PATTERN') ||
+                            key.includes('REGEX');
+                    }}
+
+                    function isSimpleFilterPattern_{widget_id}(pattern) {{
+                        return /^[\\w.*:-]+$/.test(pattern);
+                    }}
+
+                    function validateRegexPattern_{widget_id}(pattern) {{
+                        if (!pattern || isSimpleFilterPattern_{widget_id}(pattern)) {{
+                            return '';
+                        }}
+
+                        try {{
+                            new RegExp(pattern);
+                        }} catch (error) {{
+                            return error && error.message ? error.message : 'Invalid regex';
+                        }}
+                        return '';
+                    }}
+
+                    function validateRegexConfig_{widget_id}(key, raw, typeName) {{
+                        if (typeName === 'object') {{
+                            var parsed;
+                            try {{
+                                parsed = JSON.parse(raw);
+                            }} catch (error) {{
+                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
+                            }}
+                            if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {{
+                                return {{ ok: false, value: parsed, message: 'Must be a JSON object' }};
+                            }}
+                            for (var regexKey in parsed) {{
+                                var objectRegexError = validateRegexPattern_{widget_id}(regexKey);
+                                if (objectRegexError) {{
+                                    return {{ ok: false, value: parsed, message: 'Invalid regex key "' + regexKey + '": ' + objectRegexError }};
+                                }}
+                            }}
+                            return {{ ok: true, value: parsed, message: '' }};
+                        }}
+
+                        var patterns = raw.split(/[\\n,]+/).map(function(pattern) {{
+                            return pattern.trim();
+                        }}).filter(Boolean);
+                        for (var i = 0; i < patterns.length; i++) {{
+                            var regexError = validateRegexPattern_{widget_id}(patterns[i]);
+                            if (regexError) {{
+                                return {{ ok: false, value: raw, message: 'Invalid regex "' + patterns[i] + '": ' + regexError }};
+                            }}
+                        }}
+                        return {{ ok: true, value: raw, message: '' }};
+                    }}
+
+                    function validateBinaryValue_{widget_id}(raw) {{
+                        if (!raw) {{
+                            return {{ ok: true, value: raw, message: '' }};
+                        }}
+
+                        if (/['"`]/.test(raw)) {{
+                            return {{ ok: false, value: raw, message: 'Binary paths cannot contain quotes' }};
+                        }}
+
+                        if (/[;&|<>$(){{}}\\[\\]!]/.test(raw)) {{
+                            return {{ ok: false, value: raw, message: 'Binary paths can only be a binary name or absolute path' }};
+                        }}
+
+                        if (raw.startsWith('/')) {{
+                            if (/^[A-Za-z0-9_./+\\- ]+$/.test(raw)) {{
+                                return {{ ok: true, value: raw, message: '' }};
+                            }}
+                            return {{ ok: false, value: raw, message: 'Absolute paths may only contain path-safe characters' }};
+                        }}
+
+                        if (/^[A-Za-z0-9_.+-]+$/.test(raw)) {{
+                            return {{ ok: true, value: raw, message: '' }};
+                        }}
+
+                        return {{ ok: false, value: raw, message: 'Enter a binary name like wget or an absolute path like /usr/bin/wget' }};
+                    }}
+
+                    function parseValue_{widget_id}(raw) {{
+                        try {{
+                            if (raw === 'true') return true;
+                            if (raw === 'false') return false;
+                            if (raw === 'null') return null;
+                            if (raw !== '' && !isNaN(raw)) return Number(raw);
+                            if ((raw.startsWith('{{') && raw.endsWith('}}')) ||
+                                (raw.startsWith('[') && raw.endsWith(']')) ||
+                                (raw.startsWith('"') && raw.endsWith('"'))) {{
+                                return JSON.parse(raw);
+                            }}
+                        }} catch (error) {{
+                            return raw;
+                        }}
+                        return raw;
+                    }}
+
+                    function sameValue_{widget_id}(left, right) {{
+                        return left === right || JSON.stringify(left) === JSON.stringify(right);
+                    }}
+
+                    function parseTypedValue_{widget_id}(raw, typeName, meta) {{
+                        var numberValue;
+                        var parsed;
+
+                        if (typeName && meta && meta.key && isRegexConfigKey_{widget_id}(meta.key)) {{
+                            return validateRegexConfig_{widget_id}(meta.key, raw, typeName);
+                        }}
+
+                        if (typeName === 'string' && meta && meta.key && meta.key.endsWith('_BINARY')) {{
+                            return validateBinaryValue_{widget_id}(raw);
+                        }}
+
+                        if (typeName === 'string') {{
+                            if (meta.pattern) {{
+                                try {{
+                                    if (!(new RegExp(meta.pattern)).test(raw)) {{
+                                        return {{ ok: false, value: raw, message: 'Must match pattern ' + meta.pattern }};
+                                    }}
+                                }} catch (error) {{}}
+                            }}
+                            return {{ ok: true, value: raw, message: '' }};
+                        }}
+
+                        if (typeName === 'integer') {{
+                            if (!/^-?\\d+$/.test(raw)) {{
+                                return {{ ok: false, value: raw, message: 'Must be an integer' }};
+                            }}
+                            numberValue = Number(raw);
+                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
+                            }}
+                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
+                            }}
+                            return {{ ok: true, value: numberValue, message: '' }};
+                        }}
+
+                        if (typeName === 'number') {{
+                            if (raw === '' || isNaN(raw)) {{
+                                return {{ ok: false, value: raw, message: 'Must be a number' }};
+                            }}
+                            numberValue = Number(raw);
+                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
+                            }}
+                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
+                            }}
+                            return {{ ok: true, value: numberValue, message: '' }};
+                        }}
+
+                        if (typeName === 'boolean') {{
+                            var lowered = raw.toLowerCase();
+                            if (lowered === 'true' || raw === '1') return {{ ok: true, value: true, message: '' }};
+                            if (lowered === 'false' || raw === '0') return {{ ok: true, value: false, message: '' }};
+                            return {{ ok: false, value: raw, message: 'Must be true or false' }};
+                        }}
+
+                        if (typeName === 'null') {{
+                            return raw === 'null'
+                                ? {{ ok: true, value: null, message: '' }}
+                                : {{ ok: false, value: raw, message: 'Must be null' }};
+                        }}
+
+                        if (typeName === 'array' || typeName === 'object') {{
+                            try {{
+                                parsed = JSON.parse(raw);
+                            }} catch (error) {{
+                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
+                            }}
+
+                            if (typeName === 'array' && Array.isArray(parsed)) {{
+                                return {{ ok: true, value: parsed, message: '' }};
+                            }}
+                            if (typeName === 'object' && parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {{
+                                return {{ ok: true, value: parsed, message: '' }};
+                            }}
+
+                            return {{
+                                ok: false,
+                                value: parsed,
+                                message: typeName === 'array' ? 'Must be a JSON array' : 'Must be a JSON object',
+                            }};
+                        }}
+
+                        return {{ ok: true, value: parseValue_{widget_id}(raw), message: '' }};
+                    }}
+
+                    function validateValueAgainstMeta_{widget_id}(raw, meta) {{
+                        if (!meta || raw === '') {{
+                            return {{ state: 'neutral', value: raw, message: '' }};
+                        }}
+
+                        var enumValues = Array.isArray(meta.enum) ? meta.enum : [];
+                        var types = getTypes_{widget_id}(meta);
+                        if (!types.length) {{
+                            types = ['string'];
+                        }}
+
+                        var error = 'Invalid value';
+                        for (var i = 0; i < types.length; i++) {{
+                            var candidate = parseTypedValue_{widget_id}(raw, types[i], meta);
+                            if (!candidate.ok) {{
+                                error = candidate.message || error;
+                                continue;
+                            }}
+                            if (enumValues.length && !enumValues.some(function(enumValue) {{
+                                return sameValue_{widget_id}(enumValue, candidate.value) || stringifyValue_{widget_id}(enumValue) === raw;
+                            }})) {{
+                                error = 'Must be one of: ' + enumValues.map(stringifyValue_{widget_id}).join(', ');
+                                continue;
+                            }}
+                            return {{ state: 'valid', value: candidate.value, message: '' }};
+                        }}
+
+                        return {{ state: 'invalid', value: raw, message: error }};
+                    }}
+
+                    function ensureRowId_{widget_id}(row) {{
+                        if (!row.dataset.rowId) {{
+                            row.dataset.rowId = String(rowCounter_{widget_id}++);
+                        }}
+                        return row.dataset.rowId;
+                    }}
+
+                    function setRowHelp_{widget_id}(row) {{
+                        var keyInput = row.querySelector('.kv-key');
+                        var help = row.querySelector('.kv-help');
+                        if (!keyInput || !help) {{
+                            return;
+                        }}
+
+                        var key = keyInput.value.trim();
+                        if (!key) {{
+                            help.textContent = '';
+                            return;
+                        }}
+
+                        var meta = getMetaForKey_{widget_id}(key);
+                        if (meta) {{
+                            var extra = isRegexConfigKey_{widget_id}(key)
+                                ? ((meta.type === 'object' || (Array.isArray(meta.type) && meta.type.includes('object')))
+                                    ? ' Expected: JSON object with regex keys.'
+                                    : ' Expected: valid regex.')
+                                : '';
+                            var example = getExampleInput_{widget_id}(key, meta);
+                            help.textContent = [describeMeta_{widget_id}(meta) + extra, example].filter(Boolean).join(' ');
+                        }} else {{
+                            help.textContent = 'Custom key';
+                        }}
+                    }}
+
+                    function configureValueInput_{widget_id}(row) {{
+                        var keyInput = row.querySelector('.kv-key');
+                        var valueInput = row.querySelector('.kv-value');
+                        var datalist = row.querySelector('.kv-value-options');
+                        if (!keyInput || !valueInput || !datalist) {{
+                            return;
+                        }}
+
+                        var rowId = ensureRowId_{widget_id}(row);
+                        datalist.id = '{widget_id}_value_options_' + rowId;
+
+                        var meta = getMetaForKey_{widget_id}(keyInput.value.trim());
+                        var enumValues = Array.isArray(meta && meta.enum) ? meta.enum : [];
+                        var types = getTypes_{widget_id}(meta);
+                        if (!enumValues.length && types.includes('boolean')) {{
+                            enumValues = ['True', 'False'];
+                        }}
+                        if (enumValues.length) {{
+                            datalist.innerHTML = enumValues.map(function(enumValue) {{
+                                return '<option value="' + stringifyValue_{widget_id}(enumValue).replace(/"/g, '&quot;') + '"></option>';
+                            }}).join('');
+                            valueInput.setAttribute('list', datalist.id);
+                        }} else {{
+                            datalist.innerHTML = '';
+                            valueInput.removeAttribute('list');
+                        }}
+                    }}
+
+                    function setValueValidationState_{widget_id}(input, state, message) {{
+                        if (!input) {{
+                            return;
+                        }}
+
+                        if (state === 'valid') {{
+                            input.style.borderColor = '#2da44e';
+                            input.style.boxShadow = '0 0 0 1px rgba(45, 164, 78, 0.18)';
+                            input.style.backgroundColor = '#f6ffed';
+                        }} else if (state === 'invalid') {{
+                            input.style.borderColor = '#cf222e';
+                            input.style.boxShadow = '0 0 0 1px rgba(207, 34, 46, 0.18)';
+                            input.style.backgroundColor = '#fff8f8';
+                        }} else {{
+                            input.style.borderColor = '#ccc';
+                            input.style.boxShadow = 'none';
+                            input.style.backgroundColor = '';
+                        }}
+                        input.title = message || '';
+                    }}
+
+                    function applyValueValidation_{widget_id}(row) {{
+                        var keyInput = row.querySelector('.kv-key');
+                        var valueInput = row.querySelector('.kv-value');
+                        if (!keyInput || !valueInput) {{
+                            return;
+                        }}
+
+                        var key = keyInput.value.trim();
+                        if (!key) {{
+                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
+                            return;
+                        }}
+
+                        var meta = getMetaForKey_{widget_id}(key);
+                        if (!meta) {{
+                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
+                            return;
+                        }}
+
+                        var validation = validateValueAgainstMeta_{widget_id}(valueInput.value.trim(), meta);
+                        setValueValidationState_{widget_id}(valueInput, validation.state, validation.message);
+                    }}
+
+                    function coerceValueForStorage_{widget_id}(key, raw) {{
+                        var meta = getMetaForKey_{widget_id}(key);
+                        if (!meta) {{
+                            return parseValue_{widget_id}(raw);
+                        }}
+
+                        var validation = validateValueAgainstMeta_{widget_id}(raw, meta);
+                        return validation.state === 'valid' ? validation.value : raw;
+                    }}
+
+                    function initializeRows_{widget_id}() {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
+                            ensureRowId_{widget_id}(row);
+                            configureValueInput_{widget_id}(row);
+                            setRowHelp_{widget_id}(row);
+                            applyValueValidation_{widget_id}(row);
+                        }});
+                    }}
+
+                    function updateHiddenField_{widget_id}() {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        var rows = container.querySelectorAll('.key-value-row');
+                        var result = {{}};
+                        rows.forEach(function(row) {{
+                            var keyInput = row.querySelector('.kv-key');
+                            var valInput = row.querySelector('.kv-value');
+                            if (keyInput && valInput && keyInput.value.trim()) {{
+                                var key = keyInput.value.trim();
+                                var val = valInput.value.trim();
+                                result[key] = coerceValueForStorage_{widget_id}(key, val);
+                            }}
+                        }});
+                        document.getElementById('{widget_id}').value = JSON.stringify(result);
+                    }}
+
+                    window.addKeyValueRow_{widget_id} = function() {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        var newRow = document.createElement('div');
+                        newRow.className = 'key-value-row';
+                        newRow.style.cssText = 'margin-bottom: 6px;';
+                        newRow.innerHTML = '<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">' +
+                            '<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
+                            'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
+                            '<input type="text" class="kv-value" placeholder="value" ' +
+                            'style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
+                            '<datalist class="kv-value-options"></datalist>' +
+                            '<button type="button" onclick="removeKeyValueRow_{widget_id}(this)" ' +
+                            'style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>' +
+                            '</div>' +
+                            '<div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>';
+                        container.appendChild(newRow);
+                        ensureRowId_{widget_id}(newRow);
+                        configureValueInput_{widget_id}(newRow);
+                        setRowHelp_{widget_id}(newRow);
+                        applyValueValidation_{widget_id}(newRow);
+                        updateHiddenField_{widget_id}();
+                        newRow.querySelector('.kv-key').focus();
+                    }};
+
+                    window.removeKeyValueRow_{widget_id} = function(btn) {{
+                        var row = btn.closest('.key-value-row');
+                        row.remove();
+                        updateHiddenField_{widget_id}();
+                    }};
+
+                    window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
+
+                    function configRowForKey_{widget_id}(key) {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        if (!container) {{
+                            return null;
+                        }}
+                        var match = null;
+                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
+                            if (match) {{ return; }}
+                            var keyInput = row.querySelector('.kv-key');
+                            if (keyInput && keyInput.value.trim() === key) {{
+                                match = row;
+                            }}
+                        }});
+                        if (!match) {{
+                            window.addKeyValueRow_{widget_id}();
+                            var rows = container.querySelectorAll('.key-value-row');
+                            match = rows[rows.length - 1];
+                            var keyInput = match.querySelector('.kv-key');
+                            if (keyInput) {{
+                                keyInput.value = key;
+                                keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
+                            }}
+                        }}
+                        return match;
+                    }}
+
+                    function prefillConfigFromQuery_{widget_id}() {{
+                        var params = new URLSearchParams(window.location.search);
+                        var consumedConfigKeys = [];
+                        params.forEach(function(value, key) {{
+                            if (!/^[A-Z][A-Z0-9_]*$/.test(key) || !configMeta_{widget_id}[key]) {{
+                                return;
+                            }}
+                            consumedConfigKeys.push(key);
+                            var match = configRowForKey_{widget_id}(key);
+                            var valueInput = match && match.querySelector('.kv-value');
+                            if (valueInput) {{
+                                valueInput.value = value;
+                                valueInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
+                            }}
+                        }});
+                        updateHiddenField_{widget_id}();
+                        if (consumedConfigKeys.length) {{
+                            var cleanUrl = new URL(window.location.href);
+                            consumedConfigKeys.forEach(function(key) {{ cleanUrl.searchParams.delete(key); }});
+                            window.history.replaceState(null, '', cleanUrl.pathname + cleanUrl.search + cleanUrl.hash);
+                        }}
+                    }}
+
+                    function focusConfigKeyFromHash_{widget_id}() {{
+                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
+                        // to (or creates) the matching row in this editor. Used by
+                        // the setup wizard and live-config detail pages.
+                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
+                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
+                            return;
+                        }}
+                        var match = configRowForKey_{widget_id}(hash);
+                        if (!match) {{
+                            return;
+                        }}
+                        match.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
+                        var prevOutline = match.style.outline;
+                        match.style.outline = '2px solid #f59e0b';
+                        match.style.outlineOffset = '2px';
+                        match.style.transition = 'outline 1.2s ease-out';
+                        setTimeout(function() {{
+                            match.style.outline = prevOutline || 'none';
+                        }}, 1400);
+                        var valueInput = match.querySelector('.kv-value');
+                        if (valueInput) {{
+                            valueInput.focus();
+                            try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}}
+                        }}
+                    }}
+
+                    // Initialize on load
+                    document.addEventListener('DOMContentLoaded', function() {{
+                        initializeRows_{widget_id}();
+                        prefillConfigFromQuery_{widget_id}();
+                        updateHiddenField_{widget_id}();
+                        focusConfigKeyFromHash_{widget_id}();
+                    }});
+                    // Also run immediately in case DOM is already ready
+                    if (document.readyState !== 'loading') {{
+                        initializeRows_{widget_id}();
+                        prefillConfigFromQuery_{widget_id}();
+                        updateHiddenField_{widget_id}();
+                        focusConfigKeyFromHash_{widget_id}();
+                    }}
+
+                    window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id});
+
+                    // Update on any input change
+                    var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
+
+                    rowsEl_{widget_id}.addEventListener('input', function(event) {{
+                        var row = event.target.closest('.key-value-row');
+                        if (!row) {{
+                            return;
+                        }}
+
+                        if (event.target.classList.contains('kv-key')) {{
+                            configureValueInput_{widget_id}(row);
+                            setRowHelp_{widget_id}(row);
+                        }}
+
+                        if (event.target.classList.contains('kv-key') || event.target.classList.contains('kv-value')) {{
+                            applyValueValidation_{widget_id}(row);
+                            updateHiddenField_{widget_id}();
+                        }}
+                    }});
+                }})();
+            </script>
+        </div>
+        '''
+        return mark_safe(html)
+
     def _render_row(self, widget_id: str, key: str, value: str) -> str:
         from archivebox.config.common import is_sensitive_config_key
 
@@ -269,28 +872,6 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
     show_search_mode_selector = False
     change_form_template = "admin/archivebox_change_form.html"
 
-    @admin.display(description="Health", ordering="health")
-    def health_display(self, obj):
-        h = obj.health
-        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
-        return format_html('<span style="color: {};">{}</span>', color, h)
-
-    def get_ordering_fields(self, request):
-        ordering = request.GET.get("o")
-        if not ordering:
-            return set()
-        fields = set()
-        for part in ordering.split("."):
-            if not part:
-                continue
-            try:
-                idx = abs(int(part)) - 1
-            except ValueError:
-                continue
-            if 0 <= idx < len(self.list_display):
-                fields.add(self.list_display[idx])
-        return fields
-
     def get_admin_toolbar_actions(self, request, obj):
         """Return extra action button dicts for the shared change-form toolbar.
 

archivebox/base_models/models.py

@@ -4,7 +4,7 @@ __package__ = "archivebox.base_models"
 
 import json
 import shutil
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
@@ -23,10 +23,6 @@ from django_stubs_ext.db.models import TypedModelMeta
 from archivebox.config import CONSTANTS
 
 
-if TYPE_CHECKING:
-    from archivebox.config.common import ArchiveBoxBaseConfig
-
-
 def normalize_config_json_values(config: Any) -> Any:
     if not isinstance(config, dict):
         return config
@@ -59,6 +55,17 @@ def get_or_create_system_user_pk(username="system"):
     return user.pk
 
 
+class AutoDateTimeField(models.DateTimeField):
+    """DateTimeField that automatically updates on save (legacy compatibility)."""
+
+    def pre_save(self, model_instance, add):
+        if add or self.attname not in model_instance.__dict__ or not model_instance.__dict__[self.attname]:
+            value = timezone.now()
+            setattr(model_instance, self.attname, value)
+            return value
+        return super().pre_save(model_instance, add)
+
+
 class ModelWithUUID(models.Model):
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -131,7 +138,6 @@ class ModelWithHealthStats(models.Model):
 class ModelWithConfig(models.Model):
     """Mixin for models with a JSON config field."""
 
-    _runtime_config: "ArchiveBoxBaseConfig"
     config = models.JSONField(default=dict, null=True, blank=True, editable=True)
 
     class Meta(TypedModelMeta):

archivebox/cli/archivebox_add.py

@@ -388,10 +388,6 @@ def main(**kwargs):
             raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
         if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
             raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
-        if not kwargs.get("index_only"):
-            from archivebox.workers.supervisord_util import require_crawl_memory
-
-            require_crawl_memory()
 
         # Translate --only-new/--no-only-new into a crawl config override.
         # add() takes config overrides as a dict; no per-flag kwargs.

archivebox/cli/archivebox_archiveresult.py

@@ -72,7 +72,7 @@ def create_archiveresults(
     """
     from archivebox.config.common import get_config
     from archivebox.plugins.hooks import discover_hooks
-    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT
+    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
     from archivebox.core.models import Snapshot
 
     is_tty = sys.stdout.isatty()
@@ -99,9 +99,26 @@ def create_archiveresults(
         for record in records:
             record_type = record.get("type", "")
 
-            if record_type:
+            if record_type == TYPE_SNAPSHOT:
+                # Pass through the Snapshot record itself
                 pass_through_records.append(record)
-            if record.get("id") and record_type in ("", TYPE_SNAPSHOT):
+                if record.get("id"):
+                    snapshot_ids.append(record["id"])
+
+            elif record_type == TYPE_ARCHIVERESULT:
+                # ArchiveResult records: pass through if they have an id
+                if record.get("id"):
+                    pass_through_records.append(record)
+                # If no id, we could create it, but for now just pass through
+                else:
+                    pass_through_records.append(record)
+
+            elif record_type:
+                # Other typed records (Crawl, Tag, etc): pass through
+                pass_through_records.append(record)
+
+            elif record.get("id"):
+                # Untyped record with id - assume it's a snapshot ID
                 snapshot_ids.append(record["id"])
 
         # Output pass-through records first
@@ -152,20 +169,48 @@ def list_archiveresults(
     snapshot_id: str | None = None,
     limit: int | None = None,
 ) -> int:
-    """List archive results as JSONL, or formatted rows in a terminal."""
-    from archivebox.core.models import ArchiveResult
-    from archivebox.cli.cli_util import list_records, format_status
+    """
+    List ArchiveResults as JSONL with optional filters.
 
-    queryset = apply_filters(
-        ArchiveResult.objects.order_by("-start_ts"),
-        {"status": status, "plugin": plugin, "snapshot_id": snapshot_id},
-        limit=limit,
-    )
-    return list_records(
-        queryset,
-        plural="archive results",
-        render=lambda result: f"{format_status(result.status, 10)} {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
-    )
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.core.models import ArchiveResult
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = ArchiveResult.objects.all().order_by("-start_ts")
+
+    # Apply filters
+    filter_kwargs = {
+        "status": status,
+        "plugin": plugin,
+        "snapshot_id": snapshot_id,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for result in queryset:
+        if is_tty:
+            status_color = {
+                "queued": "yellow",
+                "started": "blue",
+                "succeeded": "green",
+                "failed": "red",
+                "skipped": "dim",
+                "noresults": "dim",
+                "backoff": "magenta",
+            }.get(result.status, "dim")
+            rprint(
+                f"[{status_color}]{result.status:10}[/{status_color}] {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
+            )
+        else:
+            write_record(result.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} archive results[/dim]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -173,17 +218,54 @@ def list_archiveresults(
 # =============================================================================
 
 
-def update_archiveresults(status: str | None = None) -> int:
-    """Apply supplied fields to each JSONL-selected ArchiveResult."""
+def update_archiveresults(
+    status: str | None = None,
+) -> int:
+    """
+    Update ArchiveResults from stdin JSONL.
+
+    Reads ArchiveResult records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from archivebox.misc.jsonl import read_stdin, write_record
     from archivebox.core.models import ArchiveResult
-    from archivebox.cli.cli_util import update_records
 
-    def update(archiveresult):
-        if status:
-            archiveresult.status = status
-        archiveresult.save()
+    is_tty = sys.stdout.isatty()
 
-    return update_records(ArchiveResult, update, plural="archive results")
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        result_id = record.get("id")
+        if not result_id:
+            continue
+
+        try:
+            result = ArchiveResult.objects.get(id=result_id)
+
+            # Apply updates from CLI flags
+            if status:
+                result.status = status
+
+            result.save()
+            updated_count += 1
+
+            if not is_tty:
+                write_record(result.to_json())
+
+        except ArchiveResult.DoesNotExist:
+            rprint(f"[yellow]ArchiveResult not found: {result_id}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} archive results[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -192,19 +274,52 @@ def update_archiveresults(status: str | None = None) -> int:
 
 
 def delete_archiveresults(yes: bool = False, dry_run: bool = False) -> int:
-    """Delete archive results selected by stdin JSONL; --yes confirms, --dry-run previews."""
-    from archivebox.cli.cli_util import delete_records
+    """
+    Delete ArchiveResults from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
     from archivebox.core.models import ArchiveResult
 
-    return delete_records(
-        ArchiveResult,
-        label="archive result",
-        plural="archive results",
-        preview=lambda obj: f"[dim]{obj.id}[/dim] {obj.plugin} {obj.snapshot.url[:40]}",
-        yes=yes,
-        dry_run=dry_run,
-        preview_limit=10,
-    )
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    result_ids = [r.get("id") for r in records if r.get("id")]
+
+    if not result_ids:
+        rprint("[yellow]No valid archive result IDs in input[/yellow]", file=sys.stderr)
+        return 1
+
+    results = ArchiveResult.objects.filter(id__in=result_ids)
+    count = results.count()
+
+    if count == 0:
+        rprint("[yellow]No matching archive results found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} archive results (dry run)[/yellow]", file=sys.stderr)
+        for result in results[:10]:
+            rprint(f"  [dim]{result.id}[/dim] {result.plugin} {result.snapshot.url[:40]}", file=sys.stderr)
+        if count > 10:
+            rprint(f"  ... and {count - 10} more", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = results.delete()
+    rprint(f"[green]Deleted {deleted_count} archive results[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -222,9 +337,9 @@ def main():
 @click.option("--snapshot-id", help="Snapshot ID to create results for")
 @click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
 @click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-def create_cmd(**kwargs):
+def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
     """Emit Snapshot plugin extraction requests as JSONL."""
-    sys.exit(create_archiveresults(**kwargs))
+    sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
 
 
 @main.command("list")
@@ -232,24 +347,36 @@ def create_cmd(**kwargs):
 @click.option("--plugin", "-p", help="Filter by plugin name")
 @click.option("--snapshot-id", help="Filter by snapshot ID")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(**kwargs):
+def list_cmd(
+    status: str | None,
+    plugin: str | None,
+    snapshot_id: str | None,
+    limit: int | None,
+):
     """List ArchiveResults as JSONL."""
-    sys.exit(list_archiveresults(**kwargs))
+    sys.exit(
+        list_archiveresults(
+            status=status,
+            plugin=plugin,
+            snapshot_id=snapshot_id,
+            limit=limit,
+        ),
+    )
 
 
 @main.command("update")
 @click.option("--status", "-s", help="Set status")
-def update_cmd(**kwargs):
+def update_cmd(status: str | None):
     """Update ArchiveResults from stdin JSONL."""
-    sys.exit(update_archiveresults(**kwargs))
+    sys.exit(update_archiveresults(status=status))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(**kwargs):
+def delete_cmd(yes: bool, dry_run: bool):
     """Delete ArchiveResults from stdin JSONL."""
-    sys.exit(delete_archiveresults(**kwargs))
+    sys.exit(delete_archiveresults(yes=yes, dry_run=dry_run))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_binary.py

@@ -114,20 +114,37 @@ def list_binaries(
     version__icontains: str | None = None,
     limit: int | None = None,
 ) -> int:
-    """List binaries as JSONL, or formatted rows in a terminal."""
-    from archivebox.machine.models import Binary
-    from archivebox.cli.cli_util import list_records
+    """
+    List Binaries as JSONL with optional filters.
 
-    queryset = apply_filters(
-        Binary.objects.order_by("name", "-modified_at", "-created_at"),
-        {"name": name, "abspath__icontains": abspath__icontains, "version__icontains": version__icontains},
-        limit=limit,
-    )
-    return list_records(
-        queryset,
-        plural="binaries",
-        render=lambda binary: f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}",
-    )
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.machine.models import Binary
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Binary.objects.all().order_by("name", "-modified_at", "-created_at")
+
+    # Apply filters
+    filter_kwargs = {
+        "name": name,
+        "abspath__icontains": abspath__icontains,
+        "version__icontains": version__icontains,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for binary in queryset:
+        if is_tty:
+            rprint(f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}")
+        else:
+            write_record(binary.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} binaries[/dim]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -135,19 +152,57 @@ def list_binaries(
 # =============================================================================
 
 
-def update_binaries(version: str | None = None, abspath: str | None = None) -> int:
-    """Apply supplied fields to each JSONL-selected Binary."""
+def update_binaries(
+    version: str | None = None,
+    abspath: str | None = None,
+) -> int:
+    """
+    Update Binaries from stdin JSONL.
+
+    Reads Binary records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from archivebox.misc.jsonl import read_stdin, write_record
     from archivebox.machine.models import Binary
-    from archivebox.cli.cli_util import update_records
 
-    def update(binary):
-        if version:
-            binary.version = version
-        if abspath:
-            binary.abspath = abspath
-        binary.save()
+    is_tty = sys.stdout.isatty()
 
-    return update_records(Binary, update, plural="binaries")
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        binary_id = record.get("id")
+        if not binary_id:
+            continue
+
+        try:
+            binary = Binary.objects.get(id=binary_id)
+
+            # Apply updates from CLI flags
+            if version:
+                binary.version = version
+            if abspath:
+                binary.abspath = abspath
+
+            binary.save()
+            updated_count += 1
+
+            if not is_tty:
+                write_record(binary.to_json())
+
+        except Binary.DoesNotExist:
+            rprint(f"[yellow]Binary not found: {binary_id}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} binaries[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -156,18 +211,50 @@ def update_binaries(version: str | None = None, abspath: str | None = None) -> i
 
 
 def delete_binaries(yes: bool = False, dry_run: bool = False) -> int:
-    """Delete binaries selected by stdin JSONL; --yes confirms, --dry-run previews."""
-    from archivebox.cli.cli_util import delete_records
+    """
+    Delete Binaries from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
     from archivebox.machine.models import Binary
 
-    return delete_records(
-        Binary,
-        label="binary",
-        plural="binaries",
-        preview=lambda obj: f"{obj.name} {obj.abspath}",
-        yes=yes,
-        dry_run=dry_run,
-    )
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    binary_ids = [r.get("id") for r in records if r.get("id")]
+
+    if not binary_ids:
+        rprint("[yellow]No valid binary IDs in input[/yellow]", file=sys.stderr)
+        return 1
+
+    binaries = Binary.objects.filter(id__in=binary_ids)
+    count = binaries.count()
+
+    if count == 0:
+        rprint("[yellow]No matching binaries found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} binaries (dry run)[/yellow]", file=sys.stderr)
+        for binary in binaries:
+            rprint(f"  {binary.name} {binary.abspath}", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = binaries.delete()
+    rprint(f"[green]Deleted {deleted_count} binaries[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -185,9 +272,9 @@ def main():
 @click.option("--name", "-n", required=True, help="Binary name (e.g., chrome, wget)")
 @click.option("--abspath", "-p", required=True, help="Absolute path to binary")
 @click.option("--version", "-v", default="", help="Binary version")
-def create_cmd(**kwargs):
+def create_cmd(name: str, abspath: str, version: str):
     """Create/register a Binary."""
-    sys.exit(create_binary(**kwargs))
+    sys.exit(create_binary(name=name, abspath=abspath, version=version))
 
 
 @main.command("list")
@@ -195,25 +282,37 @@ def create_cmd(**kwargs):
 @click.option("--abspath__icontains", help="Filter by path contains")
 @click.option("--version__icontains", help="Filter by version contains")
 @click.option("--limit", type=int, help="Limit number of results")
-def list_cmd(**kwargs):
+def list_cmd(
+    name: str | None,
+    abspath__icontains: str | None,
+    version__icontains: str | None,
+    limit: int | None,
+):
     """List Binaries as JSONL."""
-    sys.exit(list_binaries(**kwargs))
+    sys.exit(
+        list_binaries(
+            name=name,
+            abspath__icontains=abspath__icontains,
+            version__icontains=version__icontains,
+            limit=limit,
+        ),
+    )
 
 
 @main.command("update")
 @click.option("--version", "-v", help="Set version")
 @click.option("--abspath", "-p", help="Set path")
-def update_cmd(**kwargs):
+def update_cmd(version: str | None, abspath: str | None):
     """Update Binaries from stdin JSONL."""
-    sys.exit(update_binaries(**kwargs))
+    sys.exit(update_binaries(version=version, abspath=abspath))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(**kwargs):
+def delete_cmd(yes: bool, dry_run: bool):
     """Delete Binaries from stdin JSONL."""
-    sys.exit(delete_binaries(**kwargs))
+    sys.exit(delete_binaries(yes=yes, dry_run=dry_run))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_crawl.py

@@ -177,20 +177,43 @@ def list_crawls(
     max_depth: int | None = None,
     limit: int | None = None,
 ) -> int:
-    """List crawls as JSONL, or formatted rows in a terminal."""
-    from archivebox.crawls.models import Crawl
-    from archivebox.cli.cli_util import list_records, format_status
+    """
+    List Crawls as JSONL with optional filters.
 
-    queryset = apply_filters(
-        Crawl.objects.order_by("-created_at"),
-        {"status": status, "urls__icontains": urls__icontains, "max_depth": max_depth},
-        limit=limit,
-    )
-    return list_records(
-        queryset,
-        plural="crawls",
-        render=lambda crawl: f"{format_status(crawl.status, 8)} [dim]{crawl.id}[/dim] {crawl.urls[:50].replace(chr(10), chr(32))}...",
-    )
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.crawls.models import Crawl
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Crawl.objects.all().order_by("-created_at")
+
+    # Apply filters
+    filter_kwargs = {
+        "status": status,
+        "urls__icontains": urls__icontains,
+        "max_depth": max_depth,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for crawl in queryset:
+        if is_tty:
+            status_color = {
+                "queued": "yellow",
+                "started": "blue",
+                "sealed": "green",
+            }.get(crawl.status, "dim")
+            url_preview = crawl.urls[:50].replace("\n", " ")
+            rprint(f"[{status_color}]{crawl.status:8}[/{status_color}] [dim]{crawl.id}[/dim] {url_preview}...")
+        else:
+            write_record(crawl.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} crawls[/dim]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -198,26 +221,73 @@ def list_crawls(
 # =============================================================================
 
 
-def update_crawls(status: str | None = None, max_depth: int | None = None) -> int:
-    """Update JSONL-selected crawls through their lifecycle operations."""
+def update_crawls(
+    status: str | None = None,
+    max_depth: int | None = None,
+) -> int:
+    """
+    Update Crawls from stdin JSONL.
+
+    Reads Crawl records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
     from django.utils import timezone
+
+    from archivebox.misc.jsonl import read_stdin, write_record
     from archivebox.crawls.models import Crawl
-    from archivebox.cli.cli_util import update_records, update_record_status
 
-    def update(crawl):
-        if status:
-            try:
-                update_record_status(crawl, status)
-            except ValueError as err:
-                rprint(f"[red]{err}[/red]", file=sys.stderr)
-                return False
-        if max_depth is not None:
-            crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
-            crawl.max_depth = max_depth
-        elif not status:
-            crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
+    is_tty = sys.stdout.isatty()
 
-    return update_records(Crawl, update, plural="crawls", refresh=True)
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        crawl_id = record.get("id")
+        if not crawl_id:
+            continue
+
+        try:
+            crawl = Crawl.objects.get(id=crawl_id)
+
+            if status:
+                if status not in Crawl.StatusChoices.values:
+                    rprint(f"[red]Invalid crawl status: {status}[/red]", file=sys.stderr)
+                    continue
+                if status == Crawl.StatusChoices.SEALED:
+                    crawl.cancel()
+                elif status == Crawl.StatusChoices.PAUSED:
+                    crawl.pause()
+                elif status == Crawl.StatusChoices.QUEUED:
+                    if crawl.status == Crawl.StatusChoices.PAUSED:
+                        crawl.resume()
+                    else:
+                        crawl.update_and_requeue(status=Crawl.StatusChoices.QUEUED, retry_at=timezone.now())
+                elif status == Crawl.StatusChoices.STARTED:
+                    crawl.update_and_requeue(status=Crawl.StatusChoices.STARTED, retry_at=timezone.now())
+            if max_depth is not None:
+                crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
+                crawl.max_depth = max_depth
+            elif not status:
+                crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
+            updated_count += 1
+
+            if not is_tty:
+                crawl.refresh_from_db()
+                write_record(crawl.to_json())
+
+        except Crawl.DoesNotExist:
+            rprint(f"[yellow]Crawl not found: {crawl_id}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} crawls[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -226,18 +296,51 @@ def update_crawls(status: str | None = None, max_depth: int | None = None) -> in
 
 
 def delete_crawls(yes: bool = False, dry_run: bool = False) -> int:
-    """Delete crawls selected by stdin JSONL; --yes confirms, --dry-run previews."""
-    from archivebox.cli.cli_util import delete_records
+    """
+    Delete Crawls from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
     from archivebox.crawls.models import Crawl
 
-    return delete_records(
-        Crawl,
-        label="crawl",
-        plural="crawls",
-        preview=lambda obj: f"[dim]{obj.id}[/dim] {obj.urls[:50].replace(chr(10), chr(32))}...",
-        yes=yes,
-        dry_run=dry_run,
-    )
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    crawl_ids = [r.get("id") for r in records if r.get("id")]
+
+    if not crawl_ids:
+        rprint("[yellow]No valid crawl IDs in input[/yellow]", file=sys.stderr)
+        return 1
+
+    crawls = Crawl.objects.filter(id__in=crawl_ids)
+    count = crawls.count()
+
+    if count == 0:
+        rprint("[yellow]No matching crawls found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} crawls (dry run)[/yellow]", file=sys.stderr)
+        for crawl in crawls:
+            url_preview = crawl.urls[:50].replace("\n", " ")
+            rprint(f"  [dim]{crawl.id}[/dim] {url_preview}...", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = crawls.delete()
+    rprint(f"[green]Deleted {deleted_count} crawls[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -266,25 +369,37 @@ def create_cmd(urls: tuple, depth: int, tag: str, status: str):
 @click.option("--urls__icontains", help="Filter by URLs contains")
 @click.option("--max-depth", type=int, help="Filter by max depth")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(**kwargs):
+def list_cmd(
+    status: str | None,
+    urls__icontains: str | None,
+    max_depth: int | None,
+    limit: int | None,
+):
     """List Crawls as JSONL."""
-    sys.exit(list_crawls(**kwargs))
+    sys.exit(
+        list_crawls(
+            status=status,
+            urls__icontains=urls__icontains,
+            max_depth=max_depth,
+            limit=limit,
+        ),
+    )
 
 
 @main.command("update")
 @click.option("--status", "-s", help="Set status")
 @click.option("--max-depth", type=int, help="Set max depth")
-def update_cmd(**kwargs):
+def update_cmd(status: str | None, max_depth: int | None):
     """Update Crawls from stdin JSONL."""
-    sys.exit(update_crawls(**kwargs))
+    sys.exit(update_crawls(status=status, max_depth=max_depth))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(**kwargs):
+def delete_cmd(yes: bool, dry_run: bool):
     """Delete Crawls from stdin JSONL."""
-    sys.exit(delete_crawls(**kwargs))
+    sys.exit(delete_crawls(yes=yes, dry_run=dry_run))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_extract.py

@@ -18,7 +18,7 @@ def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[
     ArchiveResult input is accepted as a convenient reference to its parent
     Snapshot and plugin. It is never reset or converted into queued work.
     """
-    from archivebox.misc.db import uuid_ref_query
+    from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.core.models import ArchiveResult, Snapshot
     from archivebox.misc.jsonl import TYPE_ARCHIVERESULT
 
@@ -36,7 +36,7 @@ def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[
             continue
 
         if record_type == TYPE_ARCHIVERESULT:
-            result = ArchiveResult.objects.filter(uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
+            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
             if result is not None:
                 requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
                 continue
@@ -50,7 +50,7 @@ def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[
         # convenience of passing an ArchiveResult ID by trying that reference
         # only after the Snapshot lookup misses.
         if snapshot is None and record_id and not record.get("url"):
-            result = ArchiveResult.objects.filter(uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
+            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
             if result is not None:
                 requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
                 continue
@@ -177,6 +177,11 @@ def run_plugins(
     return 0
 
 
+def process_archiveresult_by_id(archiveresult_id: str) -> int:
+    """Re-run the parent Snapshot plugin referenced by an ArchiveResult."""
+    return run_plugins((), records=[{"id": archiveresult_id}], wait=True)
+
+
 @click.command()
 @click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run")
 @click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")

archivebox/cli/archivebox_machine.py

@@ -25,6 +25,7 @@ __command__ = "archivebox machine"
 import sys
 
 import rich_click as click
+from rich import print as rprint
 
 from archivebox.cli.cli_util import apply_filters
 
@@ -34,21 +35,41 @@ from archivebox.cli.cli_util import apply_filters
 # =============================================================================
 
 
-def list_machines(hostname__icontains: str | None = None, os_platform: str | None = None, limit: int | None = None) -> int:
-    """List machines as JSONL, or formatted rows in a terminal."""
-    from archivebox.machine.models import Machine
-    from archivebox.cli.cli_util import list_records
+def list_machines(
+    hostname__icontains: str | None = None,
+    os_platform: str | None = None,
+    limit: int | None = None,
+) -> int:
+    """
+    List Machines as JSONL with optional filters.
 
-    queryset = apply_filters(
-        Machine.objects.order_by("-created_at"),
-        {"hostname__icontains": hostname__icontains, "os_platform": os_platform},
-        limit=limit,
-    )
-    return list_records(
-        queryset,
-        plural="machines",
-        render=lambda machine: f"[cyan]{machine.hostname:30}[/cyan] [dim]{machine.os_platform:10}[/dim] {machine.id}",
-    )
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.machine.models import Machine
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Machine.objects.all().order_by("-created_at")
+
+    # Apply filters
+    filter_kwargs = {
+        "hostname__icontains": hostname__icontains,
+        "os_platform": os_platform,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for machine in queryset:
+        if is_tty:
+            rprint(f"[cyan]{machine.hostname:30}[/cyan] [dim]{machine.os_platform:10}[/dim] {machine.id}")
+        else:
+            write_record(machine.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} machines[/dim]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -66,9 +87,15 @@ def main():
 @click.option("--hostname__icontains", help="Filter by hostname contains")
 @click.option("--os-platform", help="Filter by OS platform")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(**kwargs):
+def list_cmd(hostname__icontains: str | None, os_platform: str | None, limit: int | None):
     """List Machines as JSONL."""
-    sys.exit(list_machines(**kwargs))
+    sys.exit(
+        list_machines(
+            hostname__icontains=hostname__icontains,
+            os_platform=os_platform,
+            limit=limit,
+        ),
+    )
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_persona.py

@@ -36,10 +36,44 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_util import apply_filters, list_records
+from archivebox.cli.cli_util import apply_filters
 from archivebox.personas import importers as persona_importers
 
 
+# =============================================================================
+# Validation Helpers
+# =============================================================================
+
+
+def validate_persona_name(name: str) -> tuple[bool, str]:
+    """
+    Validate persona name to prevent path traversal attacks.
+
+    Returns:
+        (is_valid, error_message): tuple indicating if name is valid
+    """
+    if not name or not name.strip():
+        return False, "Persona name cannot be empty"
+
+    # Check for path separators
+    if "/" in name or "\\" in name:
+        return False, "Persona name cannot contain path separators (/ or \\)"
+
+    # Check for parent directory references
+    if ".." in name:
+        return False, "Persona name cannot contain parent directory references (..)"
+
+    # Check for hidden files/directories
+    if name.startswith("."):
+        return False, "Persona name cannot start with a dot (.)"
+
+    # Ensure name doesn't contain null bytes or other dangerous chars
+    if "\x00" in name or "\n" in name or "\r" in name:
+        return False, "Persona name contains invalid characters"
+
+    return True, ""
+
+
 def ensure_path_within_personas_dir(persona_path: Path) -> bool:
     """
     Verify that a persona path is within PERSONAS_DIR.
@@ -170,7 +204,15 @@ def create_personas(
         if created:
             rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr)
         if not is_tty:
-            write_record(persona.to_json())
+            write_record(
+                {
+                    "id": str(persona.id),
+                    "name": persona.name,
+                    "path": str(persona.path),
+                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
+                    "COOKIES_FILE": persona.COOKIES_FILE,
+                },
+            )
 
     rprint(f"[green]Created {created_count} new persona(s)[/green]", file=sys.stderr)
     return 0
@@ -192,20 +234,41 @@ def list_personas(
     Exit codes:
         0: Success (even if no results)
     """
+    from archivebox.misc.jsonl import write_record
     from archivebox.personas.models import Persona
 
-    queryset = apply_filters(
-        Persona.objects.order_by("name"),
-        {"name": name, "name__icontains": name__icontains},
-        limit=limit,
-    )
+    is_tty = sys.stdout.isatty()
 
-    def render(persona):
-        cookies = "[green]✓[/green]" if persona.COOKIES_FILE else "[dim]✗[/dim]"
-        chrome = "[green]✓[/green]" if Path(persona.CHROME_USER_DATA_DIR).exists() else "[dim]✗[/dim]"
-        return f"[cyan]{persona.name:20}[/cyan] cookies:{cookies} chrome:{chrome} [dim]{persona.path}[/dim]"
+    queryset = Persona.objects.all().order_by("name")
 
-    return list_records(queryset, plural="persona(s)", render=render)
+    # Apply filters
+    filter_kwargs = {
+        "name": name,
+        "name__icontains": name__icontains,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for persona in queryset:
+        cookies_status = "[green]✓[/green]" if persona.COOKIES_FILE else "[dim]✗[/dim]"
+        chrome_status = "[green]✓[/green]" if Path(persona.CHROME_USER_DATA_DIR).exists() else "[dim]✗[/dim]"
+
+        if is_tty:
+            rprint(f"[cyan]{persona.name:20}[/cyan] cookies:{cookies_status} chrome:{chrome_status} [dim]{persona.path}[/dim]")
+        else:
+            write_record(
+                {
+                    "id": str(persona.id),
+                    "name": persona.name,
+                    "path": str(persona.path),
+                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
+                    "COOKIES_FILE": persona.COOKIES_FILE,
+                },
+            )
+        count += 1
+
+    rprint(f"[dim]Listed {count} persona(s)[/dim]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -269,7 +332,13 @@ def update_personas(name: str | None = None) -> int:
             updated_count += 1
 
             if not is_tty:
-                write_record(persona.to_json())
+                write_record(
+                    {
+                        "id": str(persona.id),
+                        "name": persona.name,
+                        "path": str(persona.path),
+                    },
+                )
 
         except Persona.DoesNotExist:
             rprint(f"[yellow]Persona not found: {persona_id or old_name}[/yellow]", file=sys.stderr)
@@ -389,9 +458,9 @@ def create_cmd(names: tuple, import_from: str | None, profile: str | None, sourc
 @click.option("--name", help="Filter by exact name")
 @click.option("--name__icontains", help="Filter by name contains")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(**kwargs):
+def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
     """List Personas as JSONL."""
-    sys.exit(list_personas(**kwargs))
+    sys.exit(list_personas(name=name, name__icontains=name__icontains, limit=limit))
 
 
 @main.command("open")
@@ -408,7 +477,7 @@ def open_cmd(name: str):
     from archivebox.config.common import get_config
     from archivebox.personas.models import Persona
 
-    valid, reason = persona_importers.validate_persona_name(name)
+    valid, reason = validate_persona_name(name)
     if not valid:
         raise click.ClickException(reason)
     persona = Persona.get_or_create_named(name)
@@ -474,17 +543,17 @@ def open_cmd(name: str):
 
 @main.command("update")
 @click.option("--name", "-n", help="Set new name")
-def update_cmd(**kwargs):
+def update_cmd(name: str | None):
     """Update Personas from stdin JSONL."""
-    sys.exit(update_personas(**kwargs))
+    sys.exit(update_personas(name=name))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(**kwargs):
+def delete_cmd(yes: bool, dry_run: bool):
     """Delete Personas from stdin JSONL."""
-    sys.exit(delete_personas(**kwargs))
+    sys.exit(delete_personas(yes=yes, dry_run=dry_run))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_process.py

@@ -28,6 +28,7 @@ __command__ = "archivebox process"
 import sys
 
 import rich_click as click
+from rich import print as rprint
 
 from archivebox.cli.cli_util import apply_filters
 
@@ -37,20 +38,46 @@ from archivebox.cli.cli_util import apply_filters
 # =============================================================================
 
 
-def list_processes(binary_name: str | None = None, machine_id: str | None = None, limit: int | None = None) -> int:
-    """List processes as JSONL, or exit codes and binary names in a terminal."""
+def list_processes(
+    binary_name: str | None = None,
+    machine_id: str | None = None,
+    limit: int | None = None,
+) -> int:
+    """
+    List Processes as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
     from archivebox.machine.models import Process
-    from archivebox.cli.cli_util import list_records
 
-    def render(process):
-        name = process.binary.name if process.binary else "unknown"
-        exit_code = process.exit_code if process.exit_code is not None else "?"
-        color = "green" if process.exit_code == 0 else "red" if process.exit_code else "yellow"
-        return f"[{color}]exit={exit_code:3}[/{color}] [cyan]{name:15}[/cyan] [dim]{process.id}[/dim]"
+    is_tty = sys.stdout.isatty()
 
-    queryset = Process.objects.select_related("binary", "machine").order_by("-started_at", "-created_at")
-    filters = {"binary__name": binary_name or None, "machine_id": machine_id or None}
-    return list_records(apply_filters(queryset, filters, limit=limit), plural="processes", render=render)
+    queryset = Process.objects.all().select_related("binary", "machine").order_by("-started_at", "-created_at")
+
+    # Apply filters
+    filter_kwargs = {}
+    if binary_name:
+        filter_kwargs["binary__name"] = binary_name
+    if machine_id:
+        filter_kwargs["machine_id"] = machine_id
+
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for process in queryset:
+        if is_tty:
+            binary_name_str = process.binary.name if process.binary else "unknown"
+            exit_code = process.exit_code if process.exit_code is not None else "?"
+            status_color = "green" if process.exit_code == 0 else "red" if process.exit_code else "yellow"
+            rprint(f"[{status_color}]exit={exit_code:3}[/{status_color}] [cyan]{binary_name_str:15}[/cyan] [dim]{process.id}[/dim]")
+        else:
+            write_record(process.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} processes[/dim]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -68,9 +95,15 @@ def main():
 @click.option("--binary-name", "-b", help="Filter by binary name")
 @click.option("--machine-id", "-m", help="Filter by machine ID")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(**kwargs):
+def list_cmd(binary_name: str | None, machine_id: str | None, limit: int | None):
     """List Processes as JSONL."""
-    sys.exit(list_processes(**kwargs))
+    sys.exit(
+        list_processes(
+            binary_name=binary_name,
+            machine_id=machine_id,
+            limit=limit,
+        ),
+    )
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_remove.py

@@ -4,17 +4,14 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox remove"
 
 import time
-from contextlib import nullcontext
 from pathlib import Path
 from collections.abc import Iterable
 
 import rich_click as click
-from rich.console import Console
 
 from django.db.models import QuerySet
 
 from archivebox.config import CONSTANTS
-from archivebox.config.common import get_config
 from archivebox.config.django import setup_django
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.checks import check_data_folder
@@ -23,6 +20,7 @@ from archivebox.misc.logging_util import (
     log_list_finished,
     log_removal_started,
     log_removal_finished,
+    TimedProgress,
 )
 from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
@@ -58,14 +56,16 @@ def remove(
     pattern_list = list(filter_patterns)
 
     log_list_started(pattern_list or None, filter_type)
-    progress = Console().status("Finding matching snapshots...") if get_config().SHOW_PROGRESS else nullcontext()
-    with progress:
+    timer = TimedProgress(360, prefix="      ")
+    try:
         if snapshots is None:
             snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
         # Freeze the target set up-front so a concurrent daemon writing new
         # snapshots can't extend the deletion under us, and so the cursor isn't
         # held open across the per-row deletes below.
         snapshot_pks = list(snapshots.values_list("pk", flat=True))
+    finally:
+        timer.end()
 
     if not snapshot_pks:
         log_removal_finished(0, 0)

archivebox/cli/archivebox_run.py

@@ -90,7 +90,7 @@ def process_stdin_records() -> int:
     )
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot, ArchiveResult
-    from archivebox.misc.db import uuid_ref_query
+    from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.crawls.models import Crawl
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Binary
@@ -117,7 +117,15 @@ def process_stdin_records() -> int:
 
         try:
             if record_type == TYPE_CRAWL:
-                crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
+                if record_id:
+                    # Existing crawl - re-queue
+                    try:
+                        crawl = Crawl.objects.get(id=record_id)
+                    except Crawl.DoesNotExist:
+                        crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
+                else:
+                    # New crawl - create it
+                    crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if crawl:
                     crawl.update_and_requeue(
@@ -130,8 +138,14 @@ def process_stdin_records() -> int:
                     queued_count += 1
 
             elif record_type == TYPE_SNAPSHOT or (record.get("url") and not record_type):
-                snapshot = Snapshot.objects.filter(id=record_id).first() if record_id else None
-                if snapshot is None:
+                if record_id:
+                    # Existing snapshot - re-queue
+                    try:
+                        snapshot = Snapshot.objects.get(id=record_id)
+                    except Snapshot.DoesNotExist:
+                        snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
+                else:
+                    # New snapshot - create it
                     snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if snapshot:
@@ -147,7 +161,7 @@ def process_stdin_records() -> int:
                 plugin_name = str(record.get("plugin") or "")
                 archiveresult = None
                 if not snapshot_id and record_id:
-                    archiveresult = ArchiveResult.objects.filter(uuid_ref_query("id", str(record_id))).select_related("snapshot").first()
+                    archiveresult = ArchiveResult.objects.filter(_uuid_ref_query("id", str(record_id))).select_related("snapshot").first()
                     if archiveresult:
                         snapshot_id = str(archiveresult.snapshot_id)
                         plugin_name = plugin_name or archiveresult.plugin
@@ -163,8 +177,12 @@ def process_stdin_records() -> int:
                     queued_count += 1
 
             elif record_type in {TYPE_BINARYREQUEST, TYPE_BINARY}:
-                binary = Binary.objects.filter(id=record_id).first() if record_id else None
-                if binary is None:
+                if record_id:
+                    try:
+                        binary = Binary.objects.get(id=record_id)
+                    except Binary.DoesNotExist:
+                        binary = Binary.from_json(record)
+                else:
                     binary = Binary.from_json(record)
 
                 if binary:

archivebox/cli/archivebox_server.py

@@ -363,12 +363,10 @@ def server(
     command = current_command(Process.TypeChoices.SERVER, data_dir=CONSTANTS.DATA_DIR, url=bind_url)
 
     def still_owns_runtime_stack() -> bool:
-        from django.db import connections
-
-        try:
-            return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
-        finally:
-            connections.close_all()
+        # This is polled at the log-tail cadence. Keep the autocommit
+        # connection open so SQLite can reuse its page cache between polls;
+        # reconnecting here repeatedly rereads the process table while idle.
+        return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
 
     shutdown_state = None
     try:
@@ -416,8 +414,13 @@ def server(
     except KeyboardInterrupt:
         pass
     finally:
-        if not shutdown_state or not shutdown_state.signal_name:
-            command.mark_exited()
+        from django.db import connections
+
+        try:
+            if not shutdown_state or not shutdown_state.signal_name:
+                command.mark_exited()
+        finally:
+            connections.close_all()
     print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")
 
 

archivebox/cli/archivebox_snapshot.py

@@ -144,7 +144,15 @@ def create_snapshots(
                     write_record(record)
 
                 # Input is a Crawl - get or create it, then create Snapshots for its URLs
-                crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
+                crawl = None
+                crawl_id = record.get("id")
+                if crawl_id:
+                    try:
+                        crawl = Crawl.objects.get(id=crawl_id)
+                    except Crawl.DoesNotExist:
+                        crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
+                else:
+                    crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if not crawl:
                     continue
@@ -166,7 +174,7 @@ def create_snapshots(
                         "depth": depth,
                         "status": status,
                     }
-                    snapshot = Snapshot.from_json(snapshot_record, overrides={"crawl": crawl, "created_by_id": created_by_id})
+                    snapshot = Snapshot.from_json(snapshot_record, overrides={"created_by_id": created_by_id})
                     if snapshot:
                         created_snapshots.append(snapshot)
                         if not is_tty:
@@ -360,29 +368,77 @@ def list_snapshots(
 # =============================================================================
 
 
-def update_snapshots(status: str | None = None, tag: str | None = None) -> int:
-    """Update JSONL-selected snapshots through their lifecycle operations."""
+def update_snapshots(
+    status: str | None = None,
+    tag: str | None = None,
+) -> int:
+    """
+    Update Snapshots from stdin JSONL.
+
+    Reads Snapshot records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
     from django.utils import timezone
+
+    from archivebox.misc.jsonl import read_stdin, write_record
     from archivebox.core.models import Snapshot
-    from archivebox.cli.cli_util import update_records, update_record_status
 
-    def update(snapshot):
-        if status:
-            try:
-                update_record_status(snapshot, status)
-            except ValueError as err:
-                rprint(f"[red]{err}[/red]", file=sys.stderr)
-                return False
-        if tag:
-            from archivebox.core.models import Tag
+    is_tty = sys.stdout.isatty()
 
-            tag_obj, _ = Tag.objects.get_or_create(name=tag)
-            snapshot.tags.add(tag_obj)
-            snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
-        if not status and not tag:
-            snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
 
-    return update_records(Snapshot, update, plural="snapshots", refresh=True)
+    updated_count = 0
+    for record in records:
+        snapshot_id = record.get("id")
+        if not snapshot_id:
+            continue
+
+        try:
+            snapshot = Snapshot.objects.get(id=snapshot_id)
+
+            if status:
+                if status not in Snapshot.StatusChoices.values:
+                    rprint(f"[red]Invalid snapshot status: {status}[/red]", file=sys.stderr)
+                    continue
+                if status == Snapshot.StatusChoices.SEALED:
+                    snapshot.cancel()
+                elif status == Snapshot.StatusChoices.PAUSED:
+                    snapshot.pause()
+                elif status == Snapshot.StatusChoices.QUEUED:
+                    if snapshot.status == Snapshot.StatusChoices.PAUSED:
+                        snapshot.resume()
+                    else:
+                        snapshot.update_and_requeue(status=Snapshot.StatusChoices.QUEUED, retry_at=timezone.now())
+                elif status == Snapshot.StatusChoices.STARTED:
+                    snapshot.update_and_requeue(status=Snapshot.StatusChoices.STARTED, retry_at=timezone.now())
+            if tag:
+                from archivebox.core.models import Tag
+
+                tag_obj, _ = Tag.objects.get_or_create(name=tag)
+                snapshot.tags.add(tag_obj)
+                snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
+
+            if not status and not tag:
+                snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
+            updated_count += 1
+
+            if not is_tty:
+                snapshot.refresh_from_db()
+                write_record(snapshot.to_json())
+
+        except Snapshot.DoesNotExist:
+            rprint(f"[yellow]Snapshot not found: {snapshot_id}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} snapshots[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -391,18 +447,50 @@ def update_snapshots(status: str | None = None, tag: str | None = None) -> int:
 
 
 def delete_snapshots(yes: bool = False, dry_run: bool = False) -> int:
-    """Delete snapshots selected by stdin JSONL; --yes confirms, --dry-run previews."""
-    from archivebox.cli.cli_util import delete_records
+    """
+    Delete Snapshots from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
     from archivebox.core.models import Snapshot
 
-    return delete_records(
-        Snapshot,
-        label="snapshot",
-        plural="snapshots",
-        preview=lambda obj: f"[dim]{obj.id}[/dim] {obj.url[:60]}",
-        yes=yes,
-        dry_run=dry_run,
-    )
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    snapshot_ids = [r.get("id") for r in records if r.get("id")]
+
+    if not snapshot_ids:
+        rprint("[yellow]No valid snapshot IDs in input[/yellow]", file=sys.stderr)
+        return 1
+
+    snapshots = Snapshot.objects.filter(id__in=snapshot_ids)
+    count = snapshots.count()
+
+    if count == 0:
+        rprint("[yellow]No matching snapshots found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} snapshots (dry run)[/yellow]", file=sys.stderr)
+        for snapshot in snapshots:
+            rprint(f"  [dim]{snapshot.id}[/dim] {snapshot.url[:60]}", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = snapshots.delete()
+    rprint(f"[green]Deleted {deleted_count} snapshots[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -437,17 +525,17 @@ def list_cmd(**kwargs):
 @main.command("update")
 @click.option("--status", "-s", help="Set status")
 @click.option("--tag", "-t", help="Add tag")
-def update_cmd(**kwargs):
+def update_cmd(status: str | None, tag: str | None):
     """Update Snapshots from stdin JSONL."""
-    sys.exit(update_snapshots(**kwargs))
+    sys.exit(update_snapshots(status=status, tag=tag))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(**kwargs):
+def delete_cmd(yes: bool, dry_run: bool):
     """Delete Snapshots from stdin JSONL."""
-    sys.exit(delete_snapshots(**kwargs))
+    sys.exit(delete_snapshots(yes=yes, dry_run=dry_run))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_tag.py

@@ -70,7 +70,7 @@ def create_tags(names: Iterable[str]) -> int:
         if not name:
             continue
 
-        tag, created = Tag.get_or_create_by_name(name)
+        tag, created = Tag.objects.get_or_create(name=name)
 
         if not is_tty:
             write_record(tag.to_json())
@@ -90,17 +90,42 @@ def create_tags(names: Iterable[str]) -> int:
 # =============================================================================
 
 
-def list_tags(name: str | None = None, name__icontains: str | None = None, limit: int | None = None) -> int:
-    """List tags as JSONL, or formatted rows in a terminal."""
-    from archivebox.core.models import Tag
-    from archivebox.cli.cli_util import list_records
+def list_tags(
+    name: str | None = None,
+    name__icontains: str | None = None,
+    limit: int | None = None,
+) -> int:
+    """
+    List Tags as JSONL with optional filters.
 
-    queryset = apply_filters(Tag.objects.order_by("name"), {"name": name, "name__icontains": name__icontains}, limit=limit)
-    return list_records(
-        queryset,
-        plural="tags",
-        render=lambda tag: f"[cyan]{tag.name:30}[/cyan] [dim]({tag.snapshot_set.count()} snapshots)[/dim]",
-    )
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.core.models import Tag
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Tag.objects.all().order_by("name")
+
+    # Apply filters
+    filter_kwargs = {
+        "name": name,
+        "name__icontains": name__icontains,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for tag in queryset:
+        snapshot_count = tag.snapshot_set.count()
+        if is_tty:
+            rprint(f"[cyan]{tag.name:30}[/cyan] [dim]({snapshot_count} snapshots)[/dim]")
+        else:
+            write_record(tag.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} tags[/dim]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -109,15 +134,56 @@ def list_tags(name: str | None = None, name__icontains: str | None = None, limit
 
 
 def update_tags(name: str | None = None) -> int:
-    """Apply supplied fields to each JSONL-selected Tag."""
+    """
+    Update Tags from stdin JSONL.
+
+    Reads Tag records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from archivebox.misc.jsonl import read_stdin, write_record
     from archivebox.core.models import Tag
-    from archivebox.cli.cli_util import update_records
 
-    def update(tag):
-        if name:
-            tag.rename(name)
+    is_tty = sys.stdout.isatty()
 
-    return update_records(Tag, update, plural="tags", by_name=True)
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        tag_id = record.get("id")
+        old_name = record.get("name")
+
+        if not tag_id and not old_name:
+            continue
+
+        try:
+            if tag_id:
+                tag = Tag.objects.get(id=tag_id)
+            else:
+                tag = Tag.objects.get(name=old_name)
+
+            # Apply updates from CLI flags
+            if name:
+                tag.name = name
+                tag.save()
+
+            updated_count += 1
+
+            if not is_tty:
+                write_record(tag.to_json())
+
+        except Tag.DoesNotExist:
+            rprint(f"[yellow]Tag not found: {tag_id or old_name}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} tags[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -126,19 +192,65 @@ def update_tags(name: str | None = None) -> int:
 
 
 def delete_tags(yes: bool = False, dry_run: bool = False) -> int:
-    """Delete tags selected by stdin JSONL; --yes confirms, --dry-run previews."""
-    from archivebox.cli.cli_util import delete_records
+    """
+    Delete Tags from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
     from archivebox.core.models import Tag
 
-    return delete_records(
-        Tag,
-        label="tag",
-        plural="tags",
-        preview=lambda obj: obj.name,
-        yes=yes,
-        dry_run=dry_run,
-        by_name=True,
-    )
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    # Collect tag IDs or names
+    tag_ids = []
+    tag_names = []
+    for r in records:
+        if r.get("id"):
+            tag_ids.append(r["id"])
+        elif r.get("name"):
+            tag_names.append(r["name"])
+
+    if not tag_ids and not tag_names:
+        rprint("[yellow]No valid tag IDs or names in input[/yellow]", file=sys.stderr)
+        return 1
+
+    from django.db.models import Q
+
+    query = Q()
+    if tag_ids:
+        query |= Q(id__in=tag_ids)
+    if tag_names:
+        query |= Q(name__in=tag_names)
+
+    tags = Tag.objects.filter(query)
+    count = tags.count()
+
+    if count == 0:
+        rprint("[yellow]No matching tags found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} tags (dry run)[/yellow]", file=sys.stderr)
+        for tag in tags:
+            rprint(f"  {tag.name}", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = tags.delete()
+    rprint(f"[green]Deleted {deleted_count} tags[/green]", file=sys.stderr)
+    return 0
 
 
 # =============================================================================
@@ -163,24 +275,24 @@ def create_cmd(names: tuple):
 @click.option("--name", help="Filter by exact name")
 @click.option("--name__icontains", help="Filter by name contains")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(**kwargs):
+def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
     """List Tags as JSONL."""
-    sys.exit(list_tags(**kwargs))
+    sys.exit(list_tags(name=name, name__icontains=name__icontains, limit=limit))
 
 
 @main.command("update")
 @click.option("--name", "-n", help="Set new name")
-def update_cmd(**kwargs):
+def update_cmd(name: str | None):
     """Update Tags from stdin JSONL."""
-    sys.exit(update_tags(**kwargs))
+    sys.exit(update_tags(name=name))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(**kwargs):
+def delete_cmd(yes: bool, dry_run: bool):
     """Delete Tags from stdin JSONL."""
-    sys.exit(delete_tags(**kwargs))
+    sys.exit(delete_tags(yes=yes, dry_run=dry_run))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_update.py

@@ -229,20 +229,21 @@ def update(
                 continue
             raise SystemExit(exit_code)
 
-    filters = {
-        "filter_patterns": filter_patterns,
-        "status": status,
-        "url__icontains": url__icontains,
-        "url__istartswith": url__istartswith,
-        "tag": tag,
-        "crawl_id": crawl_id,
-        "limit": limit,
-        "sort": sort,
-        "search": search,
-        "before": before,
-        "after": after,
-    }
-    is_filtered_update = any(filters.values())
+    is_filtered_update = any(
+        (
+            filter_patterns,
+            status,
+            url__icontains,
+            url__istartswith,
+            tag,
+            crawl_id,
+            limit,
+            sort,
+            search,
+            before,
+            after,
+        ),
+    )
     touched_snapshot_ids: set[str] = set()
     exit_code = 0
 
@@ -255,11 +256,33 @@ def update(
                 do_index = index_only or not migrate_only
 
                 if do_migrate:
-                    if is_filtered_update:
+                    if (
+                        filter_patterns
+                        or status
+                        or url__icontains
+                        or url__istartswith
+                        or tag
+                        or crawl_id
+                        or limit
+                        or sort
+                        or search
+                        or before
+                        or after
+                    ):
                         print("[*] Processing filtered snapshots from database...")
                         stats = process_filtered_snapshots(
-                            **filters,
+                            filter_patterns=filter_patterns,
                             filter_type=filter_type,
+                            status=status,
+                            url__icontains=url__icontains,
+                            url__istartswith=url__istartswith,
+                            tag=tag,
+                            crawl_id=crawl_id,
+                            limit=limit,
+                            sort=sort,
+                            search=search,
+                            before=before,
+                            after=after,
                             resume=resume,
                             batch_size=batch_size,
                             queue_for_archiving=True,
@@ -301,8 +324,18 @@ def update(
                         print("[*] No search indexing plugins are available, nothing to backfill.")
                     else:
                         snapshots = _build_filtered_snapshots_queryset(
-                            **filters,
+                            filter_patterns=filter_patterns,
                             filter_type=filter_type,
+                            status=status,
+                            url__icontains=url__icontains,
+                            url__istartswith=url__istartswith,
+                            tag=tag,
+                            crawl_id=crawl_id,
+                            limit=limit,
+                            sort=sort,
+                            search=search,
+                            before=before,
+                            after=after,
                             resume=resume,
                         )
                         stats = reindex_snapshots(
@@ -329,17 +362,32 @@ def update(
             resume_cmd.append("--migrate-only")
         if index_only:
             resume_cmd.append("--index-only")
-        resume_options = {
-            "batch-size": batch_size if batch_size != 500 else None,
-            "resume": exact_resume or resume,
-            "filter-type": filter_type if filter_type != "exact" else None,
-            **{key: value for key, value in filters.items() if key != "filter_patterns"},
-        }
-        for name, value in resume_options.items():
-            # Zero is a meaningful date boundary, but remains an unset limit.
-            if value or (name in {"before", "after"} and value is not None):
-                option = "crawl-id" if name == "crawl_id" else name
-                resume_cmd.extend([f"--{option}", str(value)])
+        if batch_size != 500:
+            resume_cmd.extend(["--batch-size", str(batch_size)])
+        if exact_resume or resume:
+            resume_cmd.extend(["--resume", str(exact_resume or resume)])
+        if before is not None:
+            resume_cmd.extend(["--before", str(before)])
+        if after is not None:
+            resume_cmd.extend(["--after", str(after)])
+        if filter_type != "exact":
+            resume_cmd.extend(["--filter-type", filter_type])
+        if status:
+            resume_cmd.extend(["--status", status])
+        if url__icontains:
+            resume_cmd.extend(["--url__icontains", url__icontains])
+        if url__istartswith:
+            resume_cmd.extend(["--url__istartswith", url__istartswith])
+        if tag:
+            resume_cmd.extend(["--tag", tag])
+        if crawl_id:
+            resume_cmd.extend(["--crawl-id", crawl_id])
+        if limit:
+            resume_cmd.extend(["--limit", str(limit)])
+        if sort:
+            resume_cmd.extend(["--sort", sort])
+        if search:
+            resume_cmd.extend(["--search", search])
         resume_cmd.extend(str(pattern) for pattern in filter_patterns)
         print("\n[red][X] archivebox update interrupted.[/red]")
         print("[yellow]Hint: resume this idempotent update with:[/yellow]")

archivebox/cli/archivebox_version.py

@@ -47,7 +47,13 @@ def _format_binary_abspath(
                 continue
 
             relative_str = relative.as_posix()
-            return prefix + relative_str if relative_str else prefix.rstrip("/")
+            if prefix == "./":
+                return "." if not relative_str else f"./{relative_str}"
+            if prefix == "ABXPKG_LIB_DIR/":
+                return "ABXPKG_LIB_DIR" if not relative_str else f"ABXPKG_LIB_DIR/{relative_str}"
+            if prefix == "PERSONAS_DIR/":
+                return "PERSONAS_DIR" if not relative_str else f"PERSONAS_DIR/{relative_str}"
+            return "~" if not relative_str else f"~/{relative_str}"
 
     return normalized.as_posix()
 
@@ -55,9 +61,22 @@ def _format_binary_abspath(
 def _render_binary_abspath(abspath: str):
     from rich.text import Text
 
-    for prefix, color in (("ABXPKG_LIB_DIR", "bright_blue"), ("PERSONAS_DIR", "medium_purple"), ("~", "cyan"), (".", "cyan")):
-        if abspath == prefix or abspath.startswith(prefix + "/"):
-            return Text.assemble((prefix, color), (abspath.removeprefix(prefix), "green"))
+    if abspath.startswith("ABXPKG_LIB_DIR/"):
+        return Text.assemble(("ABXPKG_LIB_DIR", "bright_blue"), (abspath.removeprefix("ABXPKG_LIB_DIR"), "green"))
+    if abspath == "ABXPKG_LIB_DIR":
+        return Text("ABXPKG_LIB_DIR", style="bright_blue")
+    if abspath.startswith("PERSONAS_DIR/"):
+        return Text.assemble(("PERSONAS_DIR", "medium_purple"), (abspath.removeprefix("PERSONAS_DIR"), "green"))
+    if abspath == "PERSONAS_DIR":
+        return Text("PERSONAS_DIR", style="medium_purple")
+    if abspath.startswith("~/"):
+        return Text.assemble(("~", "cyan"), (abspath.removeprefix("~"), "green"))
+    if abspath == "~":
+        return Text("~", style="cyan")
+    if abspath.startswith("./"):
+        return Text.assemble((".", "cyan"), (abspath.removeprefix("."), "green"))
+    if abspath == ".":
+        return Text(".", style="cyan")
     return Text(abspath, style="green")
 
 
@@ -182,6 +201,9 @@ def version(
     else:
         requested_names = {name for name in (binaries or ()) if name}
 
+    def binary_is_requested(logical_name: str, actual_name: str, display_name: str) -> bool:
+        return not requested_names or bool({logical_name, actual_name, display_name} & requested_names)
+
     def plugin_may_have_requested_binary(plugin) -> bool:
         if not requested_names:
             return True
@@ -240,6 +262,7 @@ def version(
 
     prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
     failures = []
+    seen_failures: set[str] = set()
     seen_rows: set[tuple[str, str, str, str]] = set()
 
     from archivebox.plugins.discovery import get_enabled_plugins, get_plugin_catalog
@@ -275,10 +298,10 @@ def version(
             prnt("", f"[yellow]Warning: Could not query collection binary records; resolving through abxpkg: {e}[/yellow]")
 
     declared_binary_specs: dict[str, dict[str, object]] = {}
-    plugin_binaries = []
     for plugin_name, plugin in plugins.items():
         if not plugin_may_have_requested_binary(plugin):
             continue
+        plugin_requested = bool(requested_names)
         plugin_enabled = plugin_name in enabled_plugin_names
         binary_records = get_required_binary_requests(
             plugin,
@@ -295,7 +318,8 @@ def version(
                 else actual_name
             )
             display_name = logical_name
-            plugin_binaries.append((plugin_name, plugin_enabled, display_name, binary_record))
+            if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
+                continue
             if not plugin_enabled and not requested_names:
                 continue
             if _binary_record_matches_runtime(db_binaries.get(logical_name), config.ABXPKG_LIB_DIR):
@@ -318,81 +342,156 @@ def version(
         loaded_binaries = asyncio.run(resolve_declared_binaries())
 
     rows: list[dict[str, object]] = []
+    any_rows = False
     any_available = False
     compact_paths = console.is_terminal
-    live_cm = Live(binary_dependency_table(rows), console=console, refresh_per_second=8) if console.is_terminal else None
+    live_enabled = console.is_terminal
+    live_cm = Live(binary_dependency_table(rows), console=console, refresh_per_second=8) if live_enabled else None
 
-    def emit_binary(plugin_name, plugin_enabled, display_name, binary, *, dim=False):
-        nonlocal any_available
-        valid = binary is not None
-        abspath = binary.abspath if valid else ""
-        version_str = str(binary.version or "unknown")[:15] if valid else "-"
-        provider = str(binary.binprovider or "env")[:8] if valid else "-"
-        any_available |= valid
-        if not valid and plugin_enabled and display_name not in failures:
-            failures.append(display_name)
-        row_key = _binary_row_dedupe_key(
-            display_name=display_name,
-            valid=valid,
-            version=version_str,
-            provider=provider,
-            abspath=abspath,
-        )
-        if row_key in seen_rows:
-            return
-        seen_rows.add(row_key)
-        rendered_path = "[grey53]not installed[/grey53]"
-        if valid:
-            display_path = (
-                _format_binary_abspath(
-                    abspath,
-                    pwd=Path.cwd(),
-                    lib_dir=config.ABXPKG_LIB_DIR,
-                    personas_dir=CONSTANTS.PERSONAS_DIR,
-                    home=Path.home(),
-                )
-                if compact_paths
-                else abspath
-            )
-            rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
-        rows.append(
-            {
-                "plugin": plugin_name,
-                "status": binary_dependency_status(enabled=plugin_enabled, valid=valid),
-                "binary": display_name,
-                "version": version_str,
-                "provider": provider,
-                "path": rendered_path,
-                "style": "dim" if dim or not plugin_enabled else "",
-            },
-        )
+    def emit_row(row: dict[str, object]) -> None:
+        rows.append(row)
         if live_cm is not None:
             live_cm.update(binary_dependency_table(rows), refresh=True)
 
     if live_cm is not None:
         live_cm.start()
     try:
-        for plugin_name, plugin_enabled, display_name, binary_record in plugin_binaries:
-            installed = db_binaries.get(display_name) if db_available else None
-            if not _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
-                # Resolve only enabled or explicitly requested dependencies. Cold-loading
-                # every optional provider makes status and migration checks needlessly slow.
-                if not plugin_enabled and not requested_names:
+        for plugin_name, plugin in plugins.items():
+            if not plugin_may_have_requested_binary(plugin):
+                continue
+            plugin_requested = bool(requested_names)
+            plugin_enabled = plugin_name in enabled_plugin_names
+            binary_records = get_required_binary_requests(
+                plugin,
+                plugin.config.required_binaries,
+                overrides=runtime_config,
+                derived_overrides=derived_config,
+                run_output_dir=CONSTANTS.DATA_DIR,
+            )
+            for binary_record in binary_records:
+                actual_name = str(binary_record["name"])
+                logical_name = (
+                    Path(actual_name).expanduser().name
+                    if ("/" in actual_name or "\\" in actual_name or actual_name.startswith("~"))
+                    else actual_name
+                )
+                display_name = logical_name
+                if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
                     continue
-                installed = loaded_binaries[json.dumps(binary_record, sort_keys=True, default=str)]
-            emit_binary(plugin_name, plugin_enabled, display_name, installed)
+
+                installed = db_binaries.get(logical_name) if db_available else None
+                if _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
+                    abspath = installed.abspath
+                    version_str = (installed.version or "unknown")[:15]
+                    provider = (installed.binprovider or "env")[:8]
+                    valid = True
+                elif not plugin_enabled and not requested_names:
+                    # `archivebox version` is expected to verify the active runtime, not
+                    # cold-load every optional plugin provider. Migration and status
+                    # checks often run with PLUGINS narrowed to a tiny set; resolving
+                    # disabled plugin binaries there can spend most of the command on
+                    # providers the current collection will never execute.
+                    continue
+                else:
+                    loaded = loaded_binaries[json.dumps(binary_record, sort_keys=True, default=str)]
+                    abspath = loaded.abspath if loaded is not None else ""
+                    version_str = str(loaded.version or "unknown")[:15] if loaded is not None else "unknown"
+                    provider = str(loaded.binprovider or "env")[:8] if loaded is not None else "env"
+                    valid = loaded is not None
+
+                any_rows = True
+                if valid:
+                    display_path = (
+                        _format_binary_abspath(
+                            abspath,
+                            pwd=Path.cwd(),
+                            lib_dir=config.ABXPKG_LIB_DIR,
+                            personas_dir=CONSTANTS.PERSONAS_DIR,
+                            home=Path.home(),
+                        )
+                        if compact_paths
+                        else abspath
+                    )
+                    rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
+                    any_available = True
+                else:
+                    rendered_path = "[grey53]not installed[/grey53]"
+                    if plugin_enabled and display_name not in seen_failures:
+                        failures.append(display_name)
+                        seen_failures.add(display_name)
+
+                row_key = _binary_row_dedupe_key(
+                    display_name=display_name,
+                    valid=valid,
+                    version=version_str if valid else "-",
+                    provider=provider if valid else "-",
+                    abspath=abspath,
+                )
+                if row_key in seen_rows:
+                    continue
+                seen_rows.add(row_key)
+
+                emit_row(
+                    {
+                        "plugin": plugin_name,
+                        "status": binary_dependency_status(enabled=plugin_enabled, valid=valid),
+                        "binary": display_name,
+                        "version": version_str if valid else "-",
+                        "provider": provider if valid else "-",
+                        "path": rendered_path,
+                        "style": "" if plugin_enabled else "dim",
+                    },
+                )
 
         if db_available:
             for binary_name, installed in db_binaries.items():
-                if not requested_names or binary_name in requested_names:
-                    emit_binary("(database)", True, binary_name, installed, dim=True)
+                if requested_names and binary_name not in requested_names:
+                    continue
+                abspath = installed.abspath
+                version_str = (installed.version or "unknown")[:15]
+                provider = (installed.binprovider or "env")[:8]
+                row_key = _binary_row_dedupe_key(
+                    display_name=binary_name,
+                    valid=True,
+                    version=version_str,
+                    provider=provider,
+                    abspath=abspath,
+                )
+                if row_key in seen_rows:
+                    continue
+                seen_rows.add(row_key)
+
+                display_path = (
+                    _format_binary_abspath(
+                        abspath,
+                        pwd=Path.cwd(),
+                        lib_dir=config.ABXPKG_LIB_DIR,
+                        personas_dir=CONSTANTS.PERSONAS_DIR,
+                        home=Path.home(),
+                    )
+                    if compact_paths
+                    else abspath
+                )
+                emit_row(
+                    {
+                        "plugin": "(database)",
+                        "status": binary_dependency_status(enabled=True, valid=True),
+                        "binary": binary_name,
+                        "version": version_str,
+                        "provider": provider,
+                        "path": _render_binary_abspath(display_path) if compact_paths else display_path,
+                        "style": "dim",
+                    },
+                )
+                any_rows = True
+                any_available = True
     finally:
         if live_cm is not None:
             live_cm.stop()
         else:
             prnt(binary_dependency_table(rows))
 
-    if not rows:
+    if not any_rows:
         prnt("", "[grey53]No required binaries declared for discovered plugins.[/grey53]")
     elif not any_available:
         prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")

archivebox/cli/cli_util.py

@@ -42,139 +42,3 @@ def apply_filters(queryset, filter_kwargs: dict, limit: int | None = None):
         queryset = queryset[:limit]
 
     return queryset
-
-
-def delete_records(
-    model,
-    *,
-    label: str,
-    plural: str,
-    preview,
-    yes: bool,
-    dry_run: bool,
-    preview_limit: int | None = None,
-    by_name: bool = False,
-) -> int:
-    """Delete a JSONL selection with one confirmation and dry-run contract.
-
-    Keep queryset deletion so Django's cascades and model cleanup signals run.
-    Persona deletion has additional filesystem policy and stays with that command.
-    """
-    import sys
-
-    from django.db.models import Q
-    from rich import print as rprint
-
-    from archivebox.misc.jsonl import read_stdin
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-    ids = [record["id"] for record in records if record.get("id")]
-    names = [record["name"] for record in records if by_name and not record.get("id") and record.get("name")]
-    if not ids and not names:
-        rprint(f"[yellow]No valid {label} IDs{' or names' if by_name else ''} in input[/yellow]", file=sys.stderr)
-        return 1
-    query = Q(id__in=ids)
-    if names:
-        query |= Q(name__in=names)
-    queryset = model.objects.filter(query)
-    count = queryset.count()
-    if not count:
-        rprint(f"[yellow]No matching {plural} found[/yellow]", file=sys.stderr)
-        return 0
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} {plural} (dry run)[/yellow]", file=sys.stderr)
-        for obj in queryset[:preview_limit] if preview_limit is not None else queryset:
-            rprint(f"  {preview(obj)}", file=sys.stderr)
-        if preview_limit is not None and count > preview_limit:
-            rprint(f"  ... and {count - preview_limit} more", file=sys.stderr)
-        return 0
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-    deleted_count, _ = queryset.delete()
-    rprint(f"[green]Deleted {deleted_count} {plural}[/green]", file=sys.stderr)
-    return 0
-
-
-def update_record_status(record, status: str) -> None:
-    """Route CLI status changes through the model's lifecycle operations."""
-    from django.utils import timezone
-
-    if status not in record.StatusChoices.values:
-        raise ValueError(f"Invalid {record._meta.model_name} status: {status}")
-    transitions = {"sealed": record.cancel, "paused": record.pause}
-    if status == "queued" and record.is_paused:
-        record.resume()
-    elif transition := transitions.get(status):
-        transition()
-    else:
-        record.update_and_requeue(status=status, retry_at=timezone.now())
-
-
-def list_records(queryset, *, plural: str, render) -> int:
-    """Stream JSONL when piped; render rows and a count in a terminal."""
-    import sys
-    from rich import print as rprint
-    from archivebox.misc.jsonl import write_record
-
-    is_tty = sys.stdout.isatty()
-    count = 0
-    for record in queryset:
-        if is_tty:
-            rprint(render(record))
-        else:
-            write_record(record.to_json())
-        count += 1
-    rprint(f"[dim]Listed {count} {plural}[/dim]", file=sys.stderr)
-    return 0
-
-
-def format_status(status: str, width: int) -> str:
-    colors = {
-        "queued": "yellow",
-        "started": "blue",
-        "sealed": "green",
-        "succeeded": "green",
-        "failed": "red",
-        "backoff": "magenta",
-    }
-    color = colors.get(status, "dim")
-    return f"[{color}]{status:{width}}[/{color}]"
-
-
-def update_records(model, update, *, plural: str, by_name: bool = False, refresh: bool = False) -> int:
-    """Apply a command's PATCH operation to each JSONL-selected model.
-
-    Returning False from the operation skips that row. Model operations stay in
-    their commands so lifecycle changes and ordinary field saves remain explicit.
-    """
-    import sys
-    from rich import print as rprint
-    from archivebox.misc.jsonl import read_stdin, write_record
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-    is_tty = sys.stdout.isatty()
-    count = 0
-    for record in records:
-        key = "id" if record.get("id") else "name" if by_name and record.get("name") else None
-        if key is None:
-            continue
-        try:
-            obj = model.objects.get(**{key: record[key]})
-            if update(obj) is False:
-                continue
-            count += 1
-            if not is_tty:
-                if refresh:
-                    obj.refresh_from_db()
-                write_record(obj.to_json())
-        except model.DoesNotExist:
-            rprint(f"[yellow]{model.__name__} not found: {record[key]}[/yellow]", file=sys.stderr)
-    rprint(f"[green]Updated {count} {plural}[/green]", file=sys.stderr)
-    return 0

archivebox/core/admin_archiveresults.py

@@ -1,5 +1,7 @@
 __package__ = "archivebox.core"
 
+import html
+import json
 import os
 import shlex
 from functools import reduce
@@ -14,16 +16,16 @@ from django.core.exceptions import PermissionDenied, SuspiciousOperation, Valida
 from django.db.models import Count, Min, Prefetch, Q, Subquery, TextField, Window
 from django.db.models.functions import Cast
 from django.shortcuts import redirect
-from django.template.loader import render_to_string
-from django.urls import reverse
+from django.urls import resolve, reverse
+from django.utils import timezone
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django.utils.text import smart_split
 
-from archivebox.base_models.admin import card_fieldset, BaseModelAdmin
+from archivebox.base_models.admin import BaseModelAdmin
 from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.core.routes_util import build_snapshot_url
-from archivebox.core.widgets import InlineTagEditorWidget, render_copy_block
+from archivebox.core.widgets import InlineTagEditorWidget
 from archivebox.machine.env_util import env_to_shell_exports
 from archivebox.misc.logging_util import printable_filesize
 from archivebox.misc.paginators import AcceleratedPaginator
@@ -114,58 +116,302 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None, can_del
     }
 
     rows = []
+    delete_url = html.escape(reverse("admin:core_archiveresult_changelist"), quote=True)
     for idx, result in enumerate(results):
         status = result.status or "queued"
         color, bg = status_colors.get(status, ("#6b7280", "#f3f4f6"))
+        output_files = result.output_files or {}
+        if isinstance(output_files, (dict, list, tuple, set)):
+            output_file_count = len(output_files)
+        elif isinstance(output_files, str):
+            try:
+                parsed = json.loads(output_files)
+                output_file_count = len(parsed) if isinstance(parsed, (dict, list, tuple, set)) else 0
+            except (TypeError, ValueError):
+                output_file_count = 0
+        else:
+            output_file_count = 0
+        output_size = int(result.output_size or 0)
+        output_size_display = html.escape(printable_filesize(output_size))
+
+        # Get plugin icon
+        icon = get_plugin_icon(result.plugin)
+
+        # Keep each timestamp component intact while allowing one wrap between them.
+        if result.end_ts:
+            end_time = (
+                f'<span style="white-space: nowrap;">{result.end_ts:%Y-%m-%d}</span>'
+                f'<wbr> <span style="white-space: nowrap;">{result.end_ts:%H:%M:%S}</span>'
+            )
+        else:
+            end_time = "-"
+
         process = result.process_record
-        output = result.output_str_for_display() or "-"
+        process_display = "-"
+        if process:
+            process_url = html.escape(reverse("admin:machine_process_change", args=[process.id]), quote=True)
+            process_label = html.escape(get_process_link_label(process), quote=True)
+            process_display = f'''
+                <a href="{process_url}"
+                   style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
+                   title="View process">{process_label}</a>
+            '''
+
+        machine_display = "-"
+        if process and process.machine_id:
+            machine_url = html.escape(reverse("admin:machine_machine_change", args=[process.machine_id]), quote=True)
+            machine_label = html.escape(str(process.machine.hostname or ""), quote=True)
+            machine_display = f'''
+                <a href="{machine_url}"
+                   style="color: #2563eb; text-decoration: none; font-size: 12px;"
+                   title="View machine">{machine_label}</a>
+            '''
+
+        # Truncate output for display
+        full_output_raw = result.output_str_for_display() or "-"
+        output_display_raw = full_output_raw[:60]
+        if len(full_output_raw) > 60:
+            output_display_raw += "..."
+        full_output = html.escape(full_output_raw)
+        output_display = html.escape(output_display_raw)
+
+        display_cmd = build_abx_dl_display_command(result)
+        replay_cmd = build_abx_dl_replay_command(result, config=config)
+        cmd_str_escaped = html.escape(display_cmd)
+        cmd_attr = html.escape(replay_cmd, quote=True)
+
+        # Build output link - use embed_path() which checks output_files first
         embed_path = result.embed_path()
-        rows.append(
-            {
-                "result": result,
-                "status": status,
-                "color": color,
-                "bg": bg,
-                "icon": get_plugin_icon(result.plugin),
-                "file_count": result.output_file_stats()[0],
-                "output_size": int(result.output_size or 0),
-                "output_size_display": printable_filesize(int(result.output_size or 0)),
-                "end_date": result.end_ts.strftime("%Y-%m-%d") if result.end_ts else "",
-                "end_time": result.end_ts.strftime("%H:%M:%S") if result.end_ts else "",
-                "process": process,
-                "process_url": reverse("admin:machine_process_change", args=[process.id]) if process else "",
-                "process_label": get_process_link_label(process) if process else "",
-                "machine_url": reverse("admin:machine_machine_change", args=[process.machine_id]) if process and process.machine_id else "",
-                "output": output,
-                "output_preview": output[:60] + ("..." if len(output) > 60 else ""),
-                "display_command": build_abx_dl_display_command(result),
-                "replay_command": build_abx_dl_replay_command(result, config=config),
-                "output_url": build_snapshot_url(
-                    str(result.snapshot_id),
-                    embed_path if embed_path and status == "succeeded" else "",
-                    config=config,
-                ),
-                "change_url": reverse("admin:core_archiveresult_change", args=[result.id]),
-                "version": result.cmd_version or "-",
-                "pwd": str(result.pwd or "-"),
-                "dom_id": f"output_{idx}_{str(result.id)[:8]}",
-                "short_id": str(result.id)[-8:],
-            },
+        snapshot_id = str(result.snapshot_id)
+        if embed_path and result.status == "succeeded":
+            output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
+        else:
+            output_link = build_snapshot_url(snapshot_id, "", config=config)
+        output_link_attr = html.escape(output_link, quote=True)
+
+        # Get version - try cmd_version field
+        version = html.escape(str(result.cmd_version if result.cmd_version else "-"), quote=True)
+        plugin_text = html.escape(str(result.plugin or ""), quote=True)
+        status_text = html.escape(str(status), quote=True)
+        pwd_text = html.escape(str(result.pwd or "-"), quote=True)
+
+        # Unique ID for this row's expandable output
+        row_id = f"output_{idx}_{str(result.id)[:8]}"
+        delete_button = ""
+        if can_delete:
+            delete_button = f'''
+                <button type="button" data-archive-result-ids="{result.id}" data-delete-url="{delete_url}"
+                        title="Delete this output">×</button>
+            '''
+
+        rows.append(f'''
+            <tr data-output-size="{output_size}" style="border-bottom: 1px solid #f1f5f9; transition: background 0.15s;" onmouseover="this.style.background='#f8fafc'" onmouseout="this.style.background='transparent'">
+                <td style="padding: 10px 12px; white-space: nowrap;">
+                    <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
+                       style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 11px;"
+                       title="View/edit archive result">
+                        <code>{str(result.id)[-8:]}</code>
+                    </a>
+                </td>
+                <td style="padding: 10px 12px; white-space: nowrap;">
+                    <span style="display: inline-block; padding: 3px 10px; border-radius: 12px;
+                                 font-size: 11px; font-weight: 600; text-transform: uppercase;
+                                 color: {color}; background: {bg};">{status_text}</span>
+                </td>
+                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{plugin_text}">
+                    {icon}
+                </td>
+                <td class="archive-results-plugin" style="padding: 10px 12px; font-weight: 500; color: #334155;">
+                        <a href="{output_link_attr}" target="_blank"
+                           style="color: #334155; text-decoration: none;"
+                       title="View output fullscreen"
+                       onmouseover="this.style.color='#2563eb'; this.style.textDecoration='underline';"
+                       onmouseout="this.style.color='#334155'; this.style.textDecoration='none';">
+                        {plugin_text}
+                    </a>
+                </td>
+                <td class="archive-results-output" style="padding: 10px 12px; max-width: 280px;">
+                    <span onclick="document.getElementById('{row_id}').open = !document.getElementById('{row_id}').open"
+                          style="display: block; max-width: 280px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
+                                 color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px; cursor: pointer;"
+                          title="Click to expand full output">
+                        {output_display}
+                    </span>
+                </td>
+                <td class="archive-results-files" style="padding: 10px 12px; color: #64748b; font-size: 12px; text-align: right;">
+                    {output_file_count}
+                </td>
+                <td class="archive-results-size" style="padding: 10px 12px; color: #64748b; font-size: 12px; text-align: right; white-space: nowrap;">
+                    {output_size_display}
+                </td>
+                <td class="archive-results-completed" style="padding: 10px 12px; color: #64748b; font-size: 12px;">
+                    {end_time}
+                </td>
+                <td style="padding: 10px 12px; white-space: nowrap;">
+                    {process_display}
+                </td>
+                <td style="padding: 10px 12px; white-space: nowrap;">
+                    {machine_display}
+                </td>
+                <td style="padding: 10px 12px; white-space: nowrap; font-family: ui-monospace, monospace; font-size: 11px; color: #64748b;">
+                    {version}
+                </td>
+                <td class="archive-results-actions-cell">
+                    <div class="archive-results-actions">
+                        <a href="{output_link_attr}" target="_blank"
+                           title="View output">📄</a>
+                        <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
+                           title="Edit">✏️</a>
+                        {delete_button}
+                    </div>
+                </td>
+            </tr>
+            <tr style="border-bottom: 1px solid #e2e8f0;">
+                <td colspan="12" style="padding: 0 12px 10px 12px;">
+                    <details id="{row_id}" style="margin: 0;">
+                        <summary style="cursor: pointer; font-size: 11px; color: #94a3b8; user-select: none;">
+                            Details &amp; Output
+                        </summary>
+                        <div style="margin-top: 8px; padding: 10px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; max-height: 200px; overflow: auto;">
+                            <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
+                                <span style="margin-right: 16px;"><b>ID:</b> <code>{result.id!s}</code></span>
+                                <span style="margin-right: 16px;"><b>Version:</b> <code>{version}</code></span>
+                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{pwd_text}</code></span>
+                            </div>
+                            <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
+                                <b>Output:</b>
+                            </div>
+                            <pre style="margin: 0; padding: 8px; background: #1e293b; border-radius: 4px; color: #e2e8f0; font-size: 12px; white-space: pre-wrap; word-break: break-all; max-height: 120px; overflow: auto;">{full_output}</pre>
+                            <div style="font-size: 11px; color: #64748b; margin-top: 8px;">
+                                <b>Command:</b>
+                            </div>
+                            <div style="position: relative; margin: 0; padding: 8px 56px 8px 8px; background: #1e293b; border-radius: 4px;">
+                                <button type="button"
+                                        data-command="{cmd_attr}"
+                                        onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
+                                        style="position: absolute; top: 6px; right: 6px; padding: 2px 8px; border: 0; border-radius: 4px; background: #334155; color: #e2e8f0; font-size: 11px; cursor: pointer;">
+                                    Copy
+                                </button>
+                                <code title="{cmd_attr}" style="display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: #e2e8f0; font-size: 11px;">{cmd_str_escaped}</code>
+                            </div>
+                        </div>
+                    </details>
+                </td>
+            </tr>
+        ''')
+
+    total_count = results[0]._inline_total_count
+    footer = ""
+    if total_count > limit:
+        footer = f"""
+            <tr data-output-footer>
+                <td colspan="12" style="padding: 12px; text-align: center; color: #64748b; font-size: 13px; background: #f8fafc;">
+                    Showing {limit} of {total_count} results &nbsp;
+                    <a href="/admin/core/archiveresult/?snapshot__id__exact={results[0].snapshot_id if results else ""}"
+                       style="color: #2563eb;">View all →</a>
+                </td>
+            </tr>
+        """
+
+    return mark_safe(f"""
+        <div style="border: 1px solid #e2e8f0; border-radius: 8px; overflow-x: auto; overflow-y: hidden; background: #fff; width: 100%;">
+            <table class="archive-results-table" style="width: 100%; min-width: 1100px; border-collapse: collapse; font-size: 14px;">
+                <thead>
+                    <tr style="background: #f8fafc; border-bottom: 2px solid #e2e8f0;">
+                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Details</th>
+                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Status</th>
+                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; width: 32px;"></th>
+                        <th class="archive-results-plugin" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Plugin</th>
+                        <th class="archive-results-output" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Output</th>
+                        <th class="archive-results-files" style="padding: 10px 12px; text-align: right; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Files</th>
+                        <th class="archive-results-size" style="padding: 10px 12px; text-align: right; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">
+                            <button type="button" data-output-size-sort style="all: unset; cursor: pointer;">Size ↕</button>
+                        </th>
+                        <th class="archive-results-completed" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Completed</th>
+                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Process</th>
+                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Machine</th>
+                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Version</th>
+                        <th class="archive-results-actions-cell" style="text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Actions</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {"".join(rows)}
+                    {footer}
+                </tbody>
+            </table>
+        </div>
+    """)
+
+
+class ArchiveResultInline(admin.TabularInline):
+    name = "Archive Results"
+    model = ArchiveResult
+    parent_model = Snapshot
+    extra = 0
+    sort_fields = ("end_ts", "plugin", "output_str", "status", "cmd_version")
+    readonly_fields = ("id", "result_id", "completed", "command", "version")
+    fields = ("start_ts", "end_ts", *readonly_fields, "plugin", "cmd", "cmd_version", "pwd", "status", "output_str")
+    ordering = ("end_ts",)
+    show_change_link = True
+
+    def get_parent_object_from_request(self, request):
+        resolved = resolve(request.path_info)
+        try:
+            return self.parent_model.objects.get(pk=resolved.kwargs["object_id"])
+        except (self.parent_model.DoesNotExist, ValidationError):
+            return None
+
+    @admin.display(
+        description="Completed",
+        ordering="end_ts",
+    )
+    def completed(self, obj):
+        return format_html('<p style="white-space: nowrap">{}</p>', obj.end_ts.strftime("%Y-%m-%d %H:%M:%S"))
+
+    def result_id(self, obj):
+        return format_html(
+            '<a href="{}"><code style="font-size: 10px">[{}]</code></a>',
+            reverse("admin:core_archiveresult_change", args=(obj.id,)),
+            str(obj.id)[:8],
         )
 
-    return mark_safe(
-        render_to_string(
-            "admin/core/archiveresult/inline.html",
-            {
-                "rows": rows,
-                "limit": limit,
-                "total_count": results[0]._inline_total_count,
-                "snapshot_id": results[0].snapshot_id,
-                "can_delete": can_delete,
-                "delete_url": reverse("admin:core_archiveresult_changelist"),
-            },
-        ),
-    )
+    def command(self, obj):
+        return format_html("<small><code>{}</code></small>", " ".join(obj.cmd or []))
+
+    def version(self, obj):
+        return format_html("<small><code>{}</code></small>", obj.cmd_version or "-")
+
+    def get_formset(self, request, obj=None, **kwargs):
+        formset = super().get_formset(request, obj, **kwargs)
+        snapshot = self.get_parent_object_from_request(request)
+        base_fields = formset.form.base_fields
+        snapshot_output_dir = str(snapshot.output_dir) if snapshot else ""
+
+        # formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
+
+        # default values for new entries
+        base_fields["status"].initial = "succeeded"
+        base_fields["start_ts"].initial = timezone.now()
+        base_fields["end_ts"].initial = timezone.now()
+        base_fields["cmd_version"].initial = "-"
+        base_fields["pwd"].initial = snapshot_output_dir
+        base_fields["cmd"].initial = '["-"]'
+        base_fields["output_str"].initial = "Manually recorded cmd output..."
+
+        if obj is not None:
+            # hidden values for existing entries and new entries
+            base_fields["start_ts"].widget = base_fields["start_ts"].hidden_widget()
+            base_fields["end_ts"].widget = base_fields["end_ts"].hidden_widget()
+            base_fields["cmd"].widget = base_fields["cmd"].hidden_widget()
+            base_fields["pwd"].widget = base_fields["pwd"].hidden_widget()
+            base_fields["cmd_version"].widget = base_fields["cmd_version"].hidden_widget()
+        return formset
+
+    def get_readonly_fields(self, request, obj=None):
+        if obj is not None:
+            return self.readonly_fields
+        else:
+            return []
 
 
 class ArchiveResultAdmin(BaseModelAdmin):
@@ -213,14 +459,40 @@ class ArchiveResultAdmin(BaseModelAdmin):
     autocomplete_fields = ("snapshot",)
 
     fieldsets = (
-        card_fieldset("Snapshot", ("snapshot", "snapshot_info", "tags_str", "admin_actions"), wide=True),
-        card_fieldset("Plugin", ("plugin_with_icon", "process_link", "status")),
-        card_fieldset("Timing", ("start_ts", "end_ts", "created_at", "modified_at")),
-        card_fieldset("Command", ("cmd", "cmd_str", "cmd_version", "pwd")),
-        card_fieldset(
+        (
+            "Snapshot",
+            {
+                "fields": ("snapshot", "snapshot_info", "tags_str", "admin_actions"),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Plugin",
+            {
+                "fields": ("plugin_with_icon", "process_link", "status"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timing",
+            {
+                "fields": ("start_ts", "end_ts", "created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Command",
+            {
+                "fields": ("cmd", "cmd_str", "cmd_version", "pwd"),
+                "classes": ("card",),
+            },
+        ),
+        (
             "Output",
-            ("output_str", "output_json", "output_files", "output_size", "output_mimetypes", "output_summary"),
-            wide=True,
+            {
+                "fields": ("output_str", "output_json", "output_files", "output_size", "output_mimetypes", "output_summary"),
+                "classes": ("card", "wide"),
+            },
         ),
     )
 
@@ -316,7 +588,18 @@ class ArchiveResultAdmin(BaseModelAdmin):
             self.list_per_page = saved_list_per_page
 
     def get_queryset(self, request):
-        ordering_fields = self.get_ordering_fields(request)
+        ordering = request.GET.get("o")
+        ordering_fields = set()
+        if ordering:
+            for part in ordering.split("."):
+                if not part:
+                    continue
+                try:
+                    idx = abs(int(part)) - 1
+                except ValueError:
+                    continue
+                if 0 <= idx < len(self.list_display):
+                    ordering_fields.add(self.list_display[idx])
 
         qs = (
             super()
@@ -517,9 +800,26 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     @admin.display(description="Command")
     def cmd_str(self, result):
-        return render_copy_block(
-            build_abx_dl_display_command(result),
-            copy_text=build_abx_dl_replay_command(result, config=self.request.archivebox_config),
+        request = self.request
+        display_cmd = build_abx_dl_display_command(result)
+        replay_cmd = build_abx_dl_replay_command(result, config=request.archivebox_config)
+        return format_html(
+            """
+            <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
+                <button type="button"
+                        data-command="{}"
+                        onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
+                        style="position: absolute; top: 6px; right: 6px; z-index: 1; padding: 2px 8px; border: 0; border-radius: 4px; background: #e2e8f0; color: #334155; font-size: 11px; cursor: pointer;">
+                    Copy
+                </button>
+                <code title="{}" style="display: block; width: 100%; max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 8px 56px 8px 8px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; font-size: 11px; box-sizing: border-box;">
+                    {}
+                </code>
+            </div>
+            """,
+            replay_cmd,
+            replay_cmd,
+            display_cmd,
         )
 
     def output_display(self, result):

archivebox/core/admin_snapshots.py

@@ -1,8 +1,6 @@
 __package__ = "archivebox.core"
 
 import json
-from copy import copy
-from urllib.parse import urlencode
 from functools import lru_cache
 from types import SimpleNamespace
 
@@ -11,12 +9,11 @@ from django.urls import path, reverse
 from django.shortcuts import get_object_or_404, redirect
 from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed
 from django.utils import timezone
-from django.utils.html import format_html
+from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
 from django.db.models import Q, Count, Exists, F, OuterRef, Prefetch
 from django import forms
 from django.template import Template, RequestContext
-from django.template.loader import render_to_string
 from django.contrib.admin.helpers import ActionForm
 
 from archivebox.config.common import get_config
@@ -26,11 +23,11 @@ from archivebox.misc.logging_util import printable_filesize
 from archivebox.search.admin import SearchResultsAdminMixin, SearchResultsChangeList
 from archivebox.search.views import admin_snapshot_search_stream_view
 from archivebox.core.routes_util import build_snapshot_url, build_web_url
+from archivebox.core.tag_util import get_or_create_tag
 from archivebox.plugins.hooks import discover_hooks
 from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_plugins
 
-from archivebox.core.widgets import render_permissions_badge, render_snapshot_progress
-from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
+from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 
 from archivebox.core.models import Tag, Snapshot, ArchiveResult
 from archivebox.crawls.models import Crawl
@@ -41,6 +38,7 @@ from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
     PERMISSIONS_META,
     get_snapshot_permissions,
+    normalize_permissions,
 )
 from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget
 
@@ -232,12 +230,10 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
 
 
 class SnapshotChangeList(SearchResultsChangeList):
-    snapshot_status_choices = Snapshot.StatusChoices.choices
-
     def __init__(self, request, *args, **kwargs):
         super().__init__(request, *args, **kwargs)
         resolver_name = request.resolver_match.url_name
-        self.embedded_changelist = request.GET.get("_embedded") in {"crawl", "snapshot"}
+        self.embedded_changelist = request.GET.get("_embedded") == "crawl"
         self.snapshot_is_grid_view = not self.embedded_changelist and (
             resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")
         )
@@ -301,7 +297,7 @@ class SnapshotChangeList(SearchResultsChangeList):
 
     def get_results(self, request):
         super().get_results(request)
-        if request.GET.get("_embedded") in {"crawl", "snapshot"}:
+        if request.GET.get("_embedded") == "crawl":
             self.full_result_count = self.result_count
             self.show_full_result_count = True
         self._attach_archiveresult_summaries()
@@ -415,19 +411,70 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                 "classes": ("card", "actions-card"),
             },
         ),
-        card_fieldset("Snapshot", ("snapshot_summary",)),
-        card_fieldset("URL", (("url_favicon", "url"), ("title", "tags_badges")), wide=True),
-        card_fieldset("Tags", ("tags_editor", "permissions_config")),
-        card_fieldset("Status", ("status", "retry_at")),
-        card_fieldset("Timestamps", ("bookmarked_at", "created_at", "modified_at", "downloaded_at")),
-        card_fieldset("Relations", ("crawl",)),
-        card_fieldset(
+        (
+            "Snapshot",
+            {
+                "fields": ("snapshot_summary",),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "URL",
+            {
+                "fields": (("url_favicon", "url"), ("title", "tags_badges")),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Tags",
+            {
+                "fields": ("tags_editor", "permissions_config"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Status",
+            {
+                "fields": ("status", "retry_at"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": ("bookmarked_at", "created_at", "modified_at", "downloaded_at"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Relations",
+            {
+                "fields": ("crawl",),
+                "classes": ("card",),
+            },
+        ),
+        (
             "Config",
-            ("config",),
-            description='<span style="display:block; margin:-4px 0 6px; font-size:11px; line-height:1.35; color:#94a3b8;">Uses <code>Crawl.config</code> by default. Only set per-snapshot overrides here when needed.</span>',
+            {
+                "fields": ("config",),
+                "description": '<span style="display:block; margin:-4px 0 6px; font-size:11px; line-height:1.35; color:#94a3b8;">Uses <code>Crawl.config</code> by default. Only set per-snapshot overrides here when needed.</span>',
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Files",
+            {
+                "fields": ("output_dir",),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Archive Results",
+            {
+                "fields": ("archiveresults_list",),
+                "classes": ("card", "wide"),
+            },
         ),
-        card_fieldset("Files", ("output_dir",)),
-        card_fieldset("Archive Results", ("archiveresults_list",), wide=True),
     )
 
     ordering = ["-created_at"]
@@ -473,7 +520,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def changelist_view(self, request, extra_context=None):
         self.request = request
         saved_list_per_page = self.list_per_page
-        embedded_changelist = request.GET.get("_embedded") in {"crawl", "snapshot"}
+        embedded_changelist = request.GET.get("_embedded") == "crawl"
         if embedded_changelist:
             try:
                 requested_per_page = int(request.GET.get("per_page", "200"))
@@ -494,27 +541,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         finally:
             self.list_per_page = saved_list_per_page
 
-    def render_embedded_changelist(self, request, *, filters, title, default_search_mode=None):
-        """Reuse snapshot search, rows, pagination, and actions in parent admin pages."""
-        changelist_path = reverse(f"{self.admin_site.name}:core_snapshot_changelist")
-        full_url = f"{changelist_path}?{urlencode(filters)}"
-        changelist_request = copy(request)
-        changelist_request.method = "GET"
-        changelist_request.path = changelist_path
-        changelist_request.GET = request.GET.copy()
-        if default_search_mode:
-            changelist_request.GET.setdefault("search_mode", default_search_mode)
-        changelist_request.GET.update({**filters, "_embedded": "snapshot", "per_page": "200"})
-        changelist_request.POST = request.POST.copy()
-        changelist_request.POST.clear()
-        response = self.changelist_view(changelist_request)
-        context = {
-            **response.context_data,
-            "snapshot_changelist_url": full_url,
-            "snapshot_changelist_title": title,
-        }
-        return mark_safe(render_to_string("admin/core/snapshot/embedded_changelist.html", context, request=request))
-
     def get_actions(self, request):
         actions = super().get_actions(request)
         if not actions:
@@ -523,7 +549,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return actions
 
     def lookup_allowed(self, lookup, value, request=None):
-        if lookup in {"crawl__id__exact", "crawl_id__exact", "crawl_id", "crawl__schedule__id__exact"}:
+        if lookup in {"crawl__id__exact", "crawl_id__exact", "crawl_id"}:
             return True
         return super().lookup_allowed(lookup, value, request=request)
 
@@ -624,7 +650,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     def get_queryset(self, request):
         self.request = request
-        ordering_fields = self.get_ordering_fields(request)
+        ordering_fields = self._get_ordering_fields(request)
         needs_files_sort = "files" in ordering_fields
         needs_tags_sort = "tags_inline" in ordering_fields
         is_change_view = request.resolver_match.url_name == "core_snapshot_change"
@@ -688,10 +714,45 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         permissions = obj.__dict__.get("snapshot_permissions")
         if permissions is None:
             permissions = obj.permissions
-        return render_permissions_badge(
+        permissions = normalize_permissions(permissions)
+        icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
+        menu_items = format_html_join(
+            "",
+            (
+                '<button type="button" class="snapshot-permissions-menu-item{}" data-permissions="{}">'
+                '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
+                "<span>{}</span>"
+                "</button>"
+            ),
+            (
+                (
+                    " is-active" if choice_value == permissions else "",
+                    choice_value,
+                    choice_fg,
+                    choice_bg,
+                    choice_icon,
+                    choice_label,
+                )
+                for choice_value, choice_label in PERMISSIONS_CHOICES
+                for choice_icon, _choice_title, choice_fg, choice_bg in [SNAPSHOT_PERMISSION_META[choice_value]]
+            ),
+        )
+        return format_html(
+            '<span class="snapshot-permissions-quick" data-current-permissions="{}" data-permissions-url="{}">'
+            '<button type="button" class="snapshot-permissions-button snapshot-permissions-{}" title="{}" aria-label="Change snapshot permissions: {}" aria-expanded="false">'
+            '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
+            "</button>"
+            '<span class="snapshot-permissions-menu" role="menu" hidden>{}</span>'
+            "</span>",
             permissions,
-            url=reverse(f"{self.admin_site.name}:core_snapshot_set_permissions", args=[obj.pk]),
-            object_name="snapshot",
+            reverse(f"{self.admin_site.name}:core_snapshot_set_permissions", args=[obj.pk]),
+            permissions,
+            label,
+            label,
+            fg,
+            bg,
+            icon,
+            menu_items,
         )
 
     @admin.display(description="Imported Timestamp")
@@ -791,6 +852,23 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             obj.pk,
         )
 
+    def status_info(self, obj):
+        request = self.request
+        config = request.archivebox_config
+        favicon_url = build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)
+        return format_html(
+            """
+            Archived: {} ({} files {}) &nbsp; &nbsp;
+            Favicon: <img src="{}" style="height: 20px"/> &nbsp; &nbsp;
+            Extension: {} &nbsp; &nbsp;
+            """,
+            "✅" if obj.is_archived else "❌",
+            obj.num_outputs,
+            self.size(obj) or "0kb",
+            favicon_url,
+            obj.extension or "-",
+        )
+
     @admin.display(description="Archive Results")
     def archiveresults_list(self, obj):
         request = self.request
@@ -1111,7 +1189,36 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
         # For started snapshots, show progress bar
         if obj.status == "started" and stats["total"] > 0:
-            return render_snapshot_progress(stats)
+            percent = stats["percent"]
+            running = stats["running"]
+            succeeded = stats["succeeded"]
+            failed = stats["failed"]
+
+            return format_html(
+                """<div style="min-width: 90px;">
+                    <div style="display: flex; align-items: center; gap: 6px; margin-bottom: 4px;">
+                        <span class="snapshot-progress-spinner"></span>
+                        <span style="font-size: 11px; color: #64748b;">{}/{} hooks</span>
+                    </div>
+                    <div style="background: #e2e8f0; border-radius: 4px; height: 6px; overflow: hidden;">
+                        <div style="background: linear-gradient(90deg, #10b981 0%, #10b981 {}%, #ef4444 {}%, #ef4444 {}%, #3b82f6 {}%, #3b82f6 100%);
+                                    width: {}%; height: 100%; transition: width 0.3s;"></div>
+                    </div>
+                    <div style="font-size: 10px; color: #94a3b8; margin-top: 2px;">
+                        ✓{} ✗{} ⏳{}
+                    </div>
+                </div>""",
+                succeeded + failed + stats["skipped"],
+                stats["total"],
+                int(succeeded / stats["total"] * 100) if stats["total"] else 0,
+                int(succeeded / stats["total"] * 100) if stats["total"] else 0,
+                int((succeeded + failed) / stats["total"] * 100) if stats["total"] else 0,
+                int((succeeded + failed) / stats["total"] * 100) if stats["total"] else 0,
+                percent,
+                succeeded,
+                failed,
+                running,
+            )
 
         # For other statuses, show simple badge
         return format_html(
@@ -1159,12 +1266,48 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
 
     def _get_progress_stats(self, obj):
-        if "_admin_progress_stats" not in obj.__dict__:
-            obj._admin_progress_stats = obj.get_progress_stats(
-                results=self._get_prefetched_results(obj),
-                expected_total=self._get_expected_hook_total(obj),
-            )
-        return obj._admin_progress_stats
+        cached_stats = obj.__dict__.get("_admin_progress_stats")
+        if cached_stats is not None:
+            return cached_stats
+
+        results = self._get_prefetched_results(obj)
+        if results is None:
+            stats = obj.get_progress_stats()
+            expected_total = self._get_expected_hook_total(obj)
+            total = max(stats["total"], expected_total)
+            completed = stats["succeeded"] + stats["failed"] + stats.get("skipped", 0) + stats.get("noresults", 0)
+            stats["total"] = total
+            stats["pending"] = max(total - completed - stats["running"], 0)
+            stats["percent"] = int((completed / total * 100) if total > 0 else 0)
+            obj._admin_progress_stats = stats
+            return stats
+
+        expected_total = self._get_expected_hook_total(obj)
+        observed_total = len(results)
+        total = max(observed_total, expected_total)
+        succeeded = sum(1 for r in results if r.status == "succeeded")
+        failed = sum(1 for r in results if r.status == "failed")
+        running = sum(1 for r in results if r.status == "started")
+        skipped = sum(1 for r in results if r.status == "skipped")
+        noresults = sum(1 for r in results if r.status == "noresults")
+        pending = max(total - succeeded - failed - running - skipped - noresults, 0)
+        completed = succeeded + failed + skipped + noresults
+        percent = int((completed / total * 100) if total > 0 else 0)
+        is_sealed = obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED, obj.StatusChoices.PAUSED)
+        stats = {
+            "total": total,
+            "succeeded": succeeded,
+            "failed": failed,
+            "running": running,
+            "pending": pending,
+            "skipped": skipped,
+            "noresults": noresults,
+            "percent": percent,
+            "output_size": obj.output_size or 0,
+            "is_sealed": is_sealed,
+        }
+        obj._admin_progress_stats = stats
+        return stats
 
     def _get_prefetched_results(self, obj):
         if "_admin_output_results" in obj.__dict__:
@@ -1220,6 +1363,22 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             return list(prefetched_cache["tags"])
         return None
 
+    def _get_ordering_fields(self, request):
+        ordering = request.GET.get("o")
+        if not ordering:
+            return set()
+        fields = set()
+        for part in ordering.split("."):
+            if not part:
+                continue
+            try:
+                idx = abs(int(part)) - 1
+            except ValueError:
+                continue
+            if 0 <= idx < len(self.list_display):
+                fields.add(self.list_display[idx])
+        return fields
+
     @admin.display(
         description="Original URL",
         ordering="url",
@@ -1231,6 +1390,12 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             obj.url[:128],
         )
 
+    @admin.display(description="Health", ordering="health")
+    def health_display(self, obj):
+        h = obj.health
+        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
+        return format_html('<span style="color: {};">{}</span>', color, h)
+
     def grid_view(self, request, extra_context=None):
         extra_context = extra_context or {}
         extra_context["snapshot_is_grid_view"] = True
@@ -1334,7 +1499,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
         tags = []
         for name in tag_names:
-            tag, _ = Tag.get_or_create_by_name(
+            tag, _ = get_or_create_tag(
                 name,
                 created_by=request.user if request.user.is_authenticated else None,
             )

archivebox/core/admin_tags.py

@@ -11,7 +11,7 @@ from django.urls import reverse
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 
-from archivebox.base_models.admin import card_fieldset, BaseModelAdmin
+from archivebox.base_models.admin import BaseModelAdmin
 from archivebox.core.models import SnapshotTag, Tag
 from archivebox.core.tag_util import (
     TAG_HAS_SNAPSHOTS_CHOICES,
@@ -69,14 +69,44 @@ class TagAdmin(BaseModelAdmin):
     ordering = ["name", "id"]
 
     fieldsets = (
-        card_fieldset("Tag", ("name",)),
-        card_fieldset("Metadata", ("id", "created_by", "created_at", "modified_at")),
-        card_fieldset("Recent Snapshots", ("snapshots",), wide=True),
+        (
+            "Tag",
+            {
+                "fields": ("name",),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Metadata",
+            {
+                "fields": ("id", "created_by", "created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Recent Snapshots",
+            {
+                "fields": ("snapshots",),
+                "classes": ("card", "wide"),
+            },
+        ),
     )
 
     add_fieldsets = (
-        card_fieldset("Tag", ("name",), wide=True),
-        card_fieldset("Metadata", ("created_by",)),
+        (
+            "Tag",
+            {
+                "fields": ("name",),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Metadata",
+            {
+                "fields": ("created_by",),
+                "classes": ("card",),
+            },
+        ),
     )
 
     def get_fieldsets(self, request: HttpRequest, obj: Tag | None = None):

archivebox/core/forms.py

@@ -9,7 +9,6 @@ from archivebox.base_models.admin import KeyValueWidget
 from archivebox.config.common import get_config, parse_delete_after
 from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
-from archivebox.crawls.forms import URLFiltersField
 from archivebox.crawls.schedule_util import validate_schedule
 from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
 from archivebox.personas.models import Persona
@@ -91,18 +90,6 @@ def edit_string_for_tag_names(tags) -> str:
     return ", ".join(sorted(names))
 
 
-class FileSizeField(forms.CharField):
-    def to_python(self, value):
-        raw_value = super().to_python(value)
-        try:
-            size = parse_filesize_to_bytes(raw_value) if raw_value else 0
-        except ValueError as err:
-            raise forms.ValidationError(str(err)) from err
-        if size < 0:
-            raise forms.ValidationError(f"{self.label} must be 0 or a positive number of bytes.")
-        return size
-
-
 class AddLinkForm(PluginConfigFormMixin, forms.Form):
     allow_crawl_execution_config_fields = False
 
@@ -149,7 +136,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
-    crawl_max_size = FileSizeField(
+    crawl_max_size = forms.CharField(
         label="Max crawl size",
         required=False,
         initial="0",
@@ -182,7 +169,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
-    snapshot_max_size = FileSizeField(
+    snapshot_max_size = forms.CharField(
         label="Max snapshot size",
         required=False,
         initial="0",
@@ -223,7 +210,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
-    url_filters = URLFiltersField(
+    url_filters = forms.Field(
         label="URL allowlist / denylist",
         required=False,
         widget=URLFiltersWidget(source_selector='textarea[name="url"]'),
@@ -360,7 +347,23 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             cleaned_data["config"] = {}
             return cleaned_data
 
-        cleaned_data["plugins"] = [plugin for _category, field, _title in PLUGIN_GROUPS for plugin in cleaned_data.get(field, [])]
+        # Combine all plugin groups into single list
+        all_selected_plugins = []
+        for field in [
+            "main_plugins",
+            "page_setup_plugins",
+            "media_plugins",
+            "text_plugins",
+            "metadata_plugins",
+            "postprocessing_plugins",
+            "other_plugins",
+        ]:
+            selected = cleaned_data.get(field)
+            if isinstance(selected, list):
+                all_selected_plugins.extend(selected)
+
+        # Store combined list for easy access
+        cleaned_data["plugins"] = all_selected_plugins
 
         plugin_config_overrides = self.clean_plugin_config_overrides(get_config(persona=cleaned_data.get("persona")))
         custom_config = cleaned_data.get("config") or {}
@@ -377,10 +380,34 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             raise forms.ValidationError("Enter at least one valid URL.")
         return value
 
+    def clean_url_filters(self):
+        from archivebox.crawls.models import Crawl
+
+        value = self.cleaned_data.get("url_filters") or {}
+        return {
+            "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
+            "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
+            "same_domain_only": bool(value.get("same_domain_only")),
+            "subpaths_only": bool(value.get("subpaths_only")),
+            "only_new": bool(value.get("only_new")),
+        }
+
     def clean_max_urls(self):
         value = self.cleaned_data.get("max_urls")
         return int(value or 0)
 
+    def clean_crawl_max_size(self):
+        raw_value = str(self.cleaned_data.get("crawl_max_size") or "").strip()
+        if not raw_value:
+            return 0
+        try:
+            value = parse_filesize_to_bytes(raw_value)
+        except ValueError as err:
+            raise forms.ValidationError(str(err))
+        if value < 0:
+            raise forms.ValidationError("Max crawl size must be 0 or a positive number of bytes.")
+        return value
+
     def clean_crawl_timeout(self):
         return self._clean_timeout_seconds(self.cleaned_data.get("crawl_timeout"), "Max crawl time", blank_value=0)
 
@@ -412,6 +439,18 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             raise forms.ValidationError(f"{field_label} must be 0 or greater than 10 seconds.")
         return value
 
+    def clean_snapshot_max_size(self):
+        raw_value = str(self.cleaned_data.get("snapshot_max_size") or "").strip()
+        if not raw_value:
+            return 0
+        try:
+            value = parse_filesize_to_bytes(raw_value)
+        except ValueError as err:
+            raise forms.ValidationError(str(err))
+        if value < 0:
+            raise forms.ValidationError("Max snapshot size must be 0 or a positive number of bytes.")
+        return value
+
     def clean_delete_after(self):
         raw_value = str(self.cleaned_data.get("delete_after") or "0").strip() or "0"
         try:

archivebox/core/models/__init__.py

@@ -1,17 +0,0 @@
-"""Core persistence models, grouped by entity and queryset responsibility."""
-
-from .archiveresults import ArchiveResult
-from .querysets import SnapshotManager, SnapshotQuerySet, UngroupedSubquery
-from .snapshots import Snapshot, SnapshotMigrationError
-from .tags import SnapshotTag, Tag
-
-__all__ = [
-    "ArchiveResult",
-    "Snapshot",
-    "SnapshotManager",
-    "SnapshotMigrationError",
-    "SnapshotQuerySet",
-    "SnapshotTag",
-    "Tag",
-    "UngroupedSubquery",
-]

archivebox/core/models/querysets.py

@@ -1,361 +0,0 @@
-from __future__ import annotations
-
-from datetime import UTC, datetime
-from typing import Any, ClassVar
-
-from django.core.exceptions import FieldDoesNotExist
-from django.db import models
-
-from archivebox.config import CONSTANTS
-from archivebox.config.common import get_config
-from archivebox.crawls.models import Crawl
-from archivebox.misc.system import atomic_write
-from archivebox.misc.util import (
-    to_json,
-)
-
-
-class UngroupedSubquery(models.Subquery):
-    """Scalar subquery that should not be copied into the outer GROUP BY."""
-
-    def get_group_by_cols(self):
-        return []
-
-
-class SnapshotQuerySet(models.QuerySet):
-    """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
-
-    def bulk_create(self, objs, *args, **kwargs):
-        objs = list(objs)
-        missing_crawl_ids = set()
-        from archivebox.core.permissions import PERMISSIONS_VALUES
-
-        for obj in objs:
-            if isinstance(obj, self.model):
-                config = dict(obj.config or {})
-                permission = str(config.get("PERMISSIONS") or "").strip().lower()
-                if permission not in PERMISSIONS_VALUES and obj.crawl_id:
-                    crawl = getattr(obj, "crawl", None)
-                    if not getattr(crawl, "permissions", None):
-                        missing_crawl_ids.add(str(obj.crawl_id))
-
-        crawl_permissions_by_id = {}
-        if missing_crawl_ids:
-            crawl_permissions_by_id = {
-                str(crawl_id): permissions
-                for crawl_id, permissions in Crawl.objects.filter(pk__in=missing_crawl_ids).values_list("pk", "permissions")
-            }
-
-        from archivebox.misc.db import truncate_overlong_charfields
-
-        for obj in objs:
-            if isinstance(obj, self.model):
-                obj.ensure_permissions_config(crawl_permissions=crawl_permissions_by_id.get(str(obj.crawl_id)))
-                # bulk_create bypasses pre_save, so clamp CharFields here too
-                # (e.g. page titles) to stay within postgres VARCHAR limits.
-                truncate_overlong_charfields(obj)
-        return super().bulk_create(objs, *args, **kwargs)
-
-    def paged_iterator(self, chunk_size: int = 500):
-        """
-        Iterate snapshots using bounded keyset pages instead of one streaming cursor.
-
-        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
-        cursor open until the full queryset is exhausted. That is fine for
-        read-only exports, but update/migration code does filesystem work and
-        writes while iterating; a long-lived read cursor there can stretch lock
-        waits across thousands of rows. This respects the queryset's existing
-        filters, order_by(), select_related(), and prefetch_related() state; if
-        no ordering is defined, it falls back to primary-key order.
-        """
-        pk_field = self.model._meta.pk.name
-        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
-
-        ordering = []
-        for term in raw_ordering:
-            if not isinstance(term, str) or term == "?":
-                ordering = []
-                break
-            field_name = term.removeprefix("-")
-            field_name = pk_field if field_name == "pk" else field_name
-            ordering.append(f"-{field_name}" if term.startswith("-") else field_name)
-
-        ordered_field_names = [term.removeprefix("-") for term in ordering]
-        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
-        try:
-            use_keyset = any(name in unique_field_names for name in ordered_field_names) and not any(
-                self.model._meta.get_field(name).null for name in ordered_field_names
-            )
-        except (AttributeError, FieldDoesNotExist):
-            use_keyset = False
-        # Expressions, nullable/related fields and non-unique orderings retain
-        # Django's ordering through bounded offset pages.
-        if not use_keyset:
-            offset = 0
-            while batch := list(self[offset : offset + chunk_size]):
-                yield from batch
-                offset += chunk_size
-            return
-
-        last_values = None
-        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
-        while True:
-            batch_qs = self.order_by(*ordering)
-            if last_values is not None:
-                page_filter = models.Q()
-                for idx, term in enumerate(ordering):
-                    descending = term.startswith("-")
-                    field_name = term[1:] if descending else term
-                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
-                    comparison = "lt" if descending else "gt"
-                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
-                batch_qs = batch_qs.filter(page_filter)
-
-            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
-            if not batch_rows:
-                break
-
-            pk_idx = value_field_names.index(pk_field)
-            snapshot_ids = [row[pk_idx] for row in batch_rows]
-            snapshots_by_id = {snapshot.pk: snapshot for snapshot in self.filter(pk__in=snapshot_ids).order_by()}
-
-            for row in batch_rows:
-                snapshot_id = row[pk_idx]
-                snapshot = snapshots_by_id.get(snapshot_id)
-                if snapshot is not None:
-                    yield snapshot
-
-            last_values = batch_rows[-1][: len(ordered_field_names)]
-
-    # =========================================================================
-    # Filtering Methods
-    # =========================================================================
-
-    FILTER_TYPES: ClassVar[dict[str, Any]] = {
-        "exact": lambda pattern: models.Q(url=pattern),
-        "substring": lambda pattern: models.Q(url__icontains=pattern),
-        "regex": lambda pattern: models.Q(url__iregex=pattern),
-        "domain": lambda pattern: (
-            models.Q(url__istartswith=f"http://{pattern}")
-            | models.Q(url__istartswith=f"https://{pattern}")
-            | models.Q(url__istartswith=f"ftp://{pattern}")
-        ),
-        "tag": lambda pattern: models.Q(tags__name=pattern),
-        "timestamp": lambda pattern: models.Q(timestamp=pattern),
-    }
-    FILTER_TYPE_CHOICES = tuple(FILTER_TYPES)
-    FILTER_ARG_KEYS = (
-        "after",
-        "before",
-        "filter_type",
-        "filter_patterns",
-        "status",
-        "url__icontains",
-        "url__istartswith",
-        "tag",
-        "crawl_id",
-        "limit",
-        "sort",
-        "search",
-    )
-    SPECIAL_FILTER_ARG_KEYS = frozenset({"filter_patterns", "filter_type", "query", "search", "tag", "before", "after", "limit", "sort"})
-
-    def filter_by_patterns(self, patterns: list[str], filter_type: str = "exact") -> SnapshotQuerySet:
-        """Filter snapshots by URL patterns using specified filter type"""
-        from archivebox.misc.logging import stderr
-
-        q_filter = models.Q()
-        for pattern in patterns:
-            try:
-                q_filter = q_filter | self.FILTER_TYPES[filter_type](pattern)
-            except KeyError:
-                stderr()
-                stderr(f"[X] Got invalid pattern for --filter-type={filter_type}:", color="red")
-                stderr(f"    {pattern}")
-                raise SystemExit(2)
-        return self.filter(q_filter)
-
-    def search(self, **kwargs) -> SnapshotQuerySet:
-
-        from archivebox.core.snapshot_status import filter_snapshots_by_status
-        from archivebox.search.query import apply_snapshot_search
-
-        queryset = self
-        filter_patterns = tuple(str(pattern) for pattern in kwargs.get("filter_patterns") or ())
-        filter_type = kwargs.get("filter_type") or "substring"
-        query = kwargs.get("query")
-        if isinstance(query, (list, tuple)):
-            query = " ".join(str(part) for part in query)
-        query = (query or (" ".join(filter_patterns) if kwargs.get("search") else "")).strip()
-
-        field_names = {field.name for field in self.model._meta.get_fields()}
-        field_names.update(field.attname for field in self.model._meta.fields)
-        field_filters = {
-            key: value
-            for key, value in kwargs.items()
-            if value is not None and key not in self.SPECIAL_FILTER_ARG_KEYS and key.split("__", 1)[0] in field_names
-        }
-        status = field_filters.pop("status", None)
-        queryset = filter_snapshots_by_status(queryset, status)
-        if field_filters:
-            queryset = queryset.filter(**field_filters)
-        if kwargs.get("tag"):
-            queryset = queryset.filter(tags__name__iexact=kwargs["tag"])
-        if kwargs.get("before") is not None:
-            queryset = queryset.filter(bookmarked_at__lt=datetime.fromtimestamp(float(kwargs["before"]), tz=UTC))
-        if kwargs.get("after") is not None:
-            queryset = queryset.filter(bookmarked_at__gt=datetime.fromtimestamp(float(kwargs["after"]), tz=UTC))
-
-        if query:
-            queryset = apply_snapshot_search(
-                queryset,
-                query,
-                search_mode=kwargs.get("search"),
-                ordering=("-created_at",) if not kwargs.get("sort") else None,
-                max_results=kwargs.get("limit"),
-                skip_backend_when_metadata_satisfies_limit=True,
-                include_metadata_for_forced_backend=True,
-            )
-        elif filter_patterns:
-            queryset = queryset.filter_by_patterns(list(filter_patterns), filter_type)
-
-        if kwargs.get("sort"):
-            queryset = queryset.order_by(kwargs["sort"])
-        elif not queryset.query.order_by:
-            queryset = queryset.order_by("-created_at")
-
-        limit = kwargs.get("limit")
-        if limit is not None and limit > 0:
-            queryset = queryset[:limit]
-
-        return queryset
-
-    # =========================================================================
-    # Export Methods
-    # =========================================================================
-
-    def to_json(self, with_headers: bool = False) -> str:
-        """Generate JSON index from snapshots"""
-        import sys
-        from datetime import datetime
-
-        from archivebox.config import VERSION
-
-        config = get_config()
-
-        MAIN_INDEX_HEADER = (
-            {
-                "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
-                "schema": "archivebox.index.json",
-                "copyright_info": config.FOOTER_INFO,
-                "meta": {
-                    "project": "ArchiveBox",
-                    "version": VERSION,
-                    "git_sha": VERSION,
-                    "website": "https://ArchiveBox.io",
-                    "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
-                    "source": "https://github.com/ArchiveBox/ArchiveBox",
-                    "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
-                    "dependencies": {},
-                },
-            }
-            if with_headers
-            else {}
-        )
-
-        snapshot_dicts = [s.to_dict(extended=True, static_export=True) for s in self.iterator(chunk_size=500)]
-
-        if with_headers:
-            output = {
-                **MAIN_INDEX_HEADER,
-                "num_links": len(snapshot_dicts),
-                "updated": datetime.now(UTC),
-                "last_run_cmd": sys.argv,
-                "links": snapshot_dicts,
-            }
-        else:
-            output = snapshot_dicts
-        return to_json(output, indent=4, sort_keys=True)
-
-    def to_csv(self, cols: list[str] | None = None, header: bool = True, separator: str = ",", ljust: int = 0) -> str:
-        """Generate CSV output from snapshots"""
-        cols = cols or ["timestamp", "is_archived", "url"]
-        header_str = separator.join(col.ljust(ljust) for col in cols) if header else ""
-        row_strs = (s.to_csv(cols=cols, ljust=ljust, separator=separator) for s in self.iterator(chunk_size=500))
-        return "\n".join((header_str, *row_strs))
-
-    def to_html(self, with_headers: bool = True) -> str:
-        """Generate main index HTML from snapshots"""
-        from datetime import datetime
-
-        from django.template.loader import render_to_string
-
-        from archivebox.config import VERSION
-        from archivebox.config.version import get_COMMIT_HASH
-
-        config = get_config()
-
-        template = "static_index.html" if with_headers else "minimal_index.html"
-        snapshot_list = list(self.iterator(chunk_size=500))
-        manifest_records = []
-        for snapshot in snapshot_list:
-            outputs = snapshot.discover_outputs(include_filesystem_fallback=True)
-            output_paths = [str(output.get("path") or "") for output in outputs]
-            snapshot._public_preview_paths = [
-                path for preferred in ("screenshot/screenshot.png", "screenshot.png") for path in output_paths if path == preferred
-            ]
-            snapshot._public_favicon_paths = [path for path in output_paths if path in ("favicon/favicon.ico", "favicon.ico")]
-            snapshot.write_html_details()
-            if with_headers:
-                # Use the same portable schema as the JSON export. Rendering
-                # above has already populated result-count caches, archive_size
-                # reuses the sealed output_size field, and tags are prefetched.
-                manifest_records.append(snapshot.to_dict(extended=True, static_export=True))
-
-        if with_headers:
-            manifest = "".join(f"{to_json(record, indent=None, sort_keys=True)}\n" for record in manifest_records)
-            atomic_write(str(CONSTANTS.DATA_DIR / CONSTANTS.JSONL_INDEX_FILENAME), manifest)
-
-        return render_to_string(
-            template,
-            {
-                "version": VERSION,
-                "git_sha": get_COMMIT_HASH() or VERSION,
-                "num_links": str(len(snapshot_list)),
-                "date_updated": datetime.now(UTC).strftime("%Y-%m-%d"),
-                "time_updated": datetime.now(UTC).strftime("%Y-%m-%d %H:%M"),
-                "links": snapshot_list,
-                "FOOTER_INFO": config.FOOTER_INFO,
-                "STATIC_EXPORT": True,
-                "STATIC_EXPORT_DIR": CONSTANTS.DATA_DIR,
-            },
-        )
-
-
-class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ignore[unsupported-base]
-    """Manager for Snapshot model - uses SnapshotQuerySet for chainable methods"""
-
-    def filter(self, *args, **kwargs):
-        domain = kwargs.pop("domain", None)
-        qs = super().filter(*args, **kwargs)
-        if domain:
-            qs = qs.filter(url__icontains=f"://{domain}")
-        return qs
-
-    def get_queryset(self):
-        # Don't prefetch by default - it causes "too many open files" during bulk operations
-        # Views/templates can add .prefetch_related('tags', 'archiveresult_set') where needed
-        return super().get_queryset()
-
-    # =========================================================================
-    # Import Methods
-    # =========================================================================
-
-    def remove(self, atomic: bool = False) -> tuple:
-        """Remove snapshots from the database"""
-        from django.db import transaction
-
-        if atomic:
-            with transaction.atomic():
-                return self.get_queryset().delete()
-        return self.get_queryset().delete()

archivebox/core/models/tags.py

@@ -1,150 +0,0 @@
-from __future__ import annotations
-
-from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, ClassVar
-
-from django.conf import settings
-from django.db import IntegrityError, models
-from django.urls import reverse_lazy
-from django.utils import timezone
-from django.utils.text import slugify
-
-from archivebox.base_models.models import (
-    ModelWithUUID,
-    get_or_create_system_user_pk,
-)
-from archivebox.misc.util import (
-    sanitize_html_text,
-)
-
-if TYPE_CHECKING:
-    from .snapshots import Snapshot
-
-
-class Tag(ModelWithUUID):
-    # Keep AutoField for compatibility with main branch migrations
-    # Don't use UUIDField here - requires complex FK transformation
-    id = models.AutoField(primary_key=True, serialize=False, verbose_name="ID")
-    created_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL,
-        on_delete=models.CASCADE,
-        default=get_or_create_system_user_pk,
-        null=True,
-        related_name="tag_set",
-    )
-    created_at = models.DateTimeField(default=timezone.now, db_index=True, null=True)
-    modified_at = models.DateTimeField(auto_now=True)
-    name = models.CharField(unique=True, blank=False, max_length=100)
-
-    @classmethod
-    def get_or_create_by_name(
-        cls,
-        name: str,
-        *,
-        defaults: Mapping[str, Any] | None = None,
-        created_by=None,
-    ) -> tuple[Tag, bool]:
-        name = cls.normalize_name(name)
-        defaults = dict(defaults or {})
-        if created_by is not None:
-            defaults["created_by"] = created_by
-        tag = cls.objects.filter(name__iexact=name).first()
-        if tag:
-            return tag, False
-        try:
-            return cls.objects.create(name=name, **(defaults or {})), True
-        except IntegrityError:
-            tag = cls.objects.filter(name__iexact=name).first()
-            if tag is None:
-                raise
-            return tag, False
-
-    @staticmethod
-    def normalize_name(name: str) -> str:
-        name = sanitize_html_text(name).strip()
-        if not name:
-            raise ValueError("Tag name is required")
-        return name
-
-    def rename(self, name: str) -> Tag:
-        name = self.normalize_name(name)
-        existing = type(self).objects.filter(name__iexact=name).exclude(pk=self.pk).first()
-        if existing:
-            raise ValueError(f'Tag "{existing.name}" already exists')
-        if self.name != name:
-            self.name = name
-            self.save()
-        return self
-
-    snapshot_set: models.Manager[Snapshot]
-
-    class Meta(ModelWithUUID.Meta):
-        app_label = "core"
-        verbose_name = "Tag"
-        verbose_name_plural = "Tags"
-
-    def __str__(self):
-        return self.name
-
-    def save(self, *args, **kwargs):
-        update_fields = kwargs.get("update_fields")
-        if update_fields is None or "name" in update_fields:
-            self.name = sanitize_html_text(self.name).strip()
-        super().save(*args, **kwargs)
-
-    @property
-    def slug(self) -> str:
-        """ASCII-safe slugified form of the tag name (derived, not stored)."""
-        return slugify(self.name or "") or "tag"
-
-    @property
-    def api_url(self) -> str:
-        return str(reverse_lazy("api-1:get_tag", args=[self.id]))
-
-    def to_json(self) -> dict:
-        """
-        Convert Tag model instance to a JSON-serializable dict.
-        """
-        from archivebox.config import VERSION
-
-        return {
-            "type": "Tag",
-            "schema_version": VERSION,
-            "id": str(self.id),
-            "name": self.name,
-        }
-
-    @staticmethod
-    def from_json(record: dict[str, Any], overrides: dict[str, Any] | None = None):
-        """
-        Create/update Tag from JSON dict.
-
-        Args:
-            record: JSON dict with 'name' field
-            overrides: Optional dict with 'snapshot' to auto-attach tag
-
-        Returns:
-            Tag instance or None
-        """
-        name = record.get("name")
-        if not name:
-            return None
-
-        tag, _ = Tag.get_or_create_by_name(name)
-
-        # Auto-attach to snapshot if in overrides
-        if overrides and "snapshot" in overrides and tag:
-            overrides["snapshot"].add_tag_ids([tag.pk])
-
-        return tag
-
-
-class SnapshotTag(models.Model):
-    id = models.AutoField(primary_key=True)
-    snapshot = models.ForeignKey("Snapshot", db_column="snapshot_id", on_delete=models.CASCADE, to_field="id")
-    tag = models.ForeignKey(Tag, db_column="tag_id", on_delete=models.CASCADE, to_field="id")
-
-    class Meta:
-        app_label = "core"
-        db_table = "core_snapshot_tags"
-        unique_together: ClassVar[list[tuple[str, str]]] = [("snapshot", "tag")]

archivebox/core/routes_util.py

@@ -60,6 +60,13 @@ def _csrf_trusted_origins(config) -> list[str]:
     return seen
 
 
+def _allowed_hosts(config) -> set[str]:
+    raw = (config.ALLOWED_HOSTS or "").strip()
+    if not raw:
+        return set()
+    return {entry.strip().lower() for entry in raw.split(",") if entry.strip() and entry.strip() != "*"}
+
+
 def derive_base_url_from_csrf(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     """Pick a single CSRF_TRUSTED_ORIGINS entry to act as the implicit BASE_URL.
 

archivebox/core/tag_util.py

@@ -5,6 +5,7 @@ from collections import defaultdict
 from typing import Any
 from urllib.parse import unquote
 
+from django.contrib.auth.models import User
 from django.db.models import Count, Exists, F, OuterRef, QuerySet
 from django.db.models.functions import Lower
 from django.http import HttpRequest
@@ -146,6 +147,34 @@ def get_tag_by_ref(tag_ref: str | int) -> Tag:
     return Tag.objects.get(name__iexact=decoded)
 
 
+def get_or_create_tag(name: str, created_by: User | None = None) -> tuple[Tag, bool]:
+    normalized_name = normalize_tag_name(name)
+    if not normalized_name:
+        raise ValueError("Tag name is required")
+
+    defaults = {"created_by": created_by} if created_by is not None else None
+    return Tag.get_or_create_by_name(normalized_name, defaults=defaults)
+
+
+def rename_tag(tag: Tag, name: str) -> Tag:
+    normalized_name = normalize_tag_name(name)
+    if not normalized_name:
+        raise ValueError("Tag name is required")
+
+    existing = Tag.objects.filter(name__iexact=normalized_name).exclude(pk=tag.pk).first()
+    if existing:
+        raise ValueError(f'Tag "{existing.name}" already exists')
+
+    if tag.name != normalized_name:
+        tag.name = normalized_name
+        tag.save()
+    return tag
+
+
+def delete_tag(tag: Tag) -> tuple[int, dict[str, int]]:
+    return tag.delete()
+
+
 def export_tag_urls(tag: Tag) -> str:
     urls = tag.snapshot_set.order_by("-downloaded_at", "-created_at", "-pk").values_list("url", flat=True)
     return "\n".join(urls)
@@ -221,10 +250,10 @@ def build_tag_card(tag: Tag, snapshot_previews: list[dict[str, Any]] | None = No
         "num_snapshots": count,
         "filter_url": f"/admin/core/snapshot/?tags__id__exact={tag.pk}",
         "edit_url": f"/admin/core/tag/{tag.pk}/change/",
-        "export_urls_url": reverse("api-1:tag_urls_export", args=[tag.pk]),
-        "export_jsonl_url": reverse("api-1:tag_snapshots_export", args=[tag.pk]),
-        "rename_url": reverse("api-1:rename_tag", args=[tag.pk]),
-        "delete_url": reverse("api-1:delete_tag", args=[tag.pk]),
+        "export_urls_url": f"/api/v1/core/tag/{tag.pk}/urls.txt",
+        "export_jsonl_url": f"/api/v1/core/tag/{tag.pk}/snapshots.jsonl",
+        "rename_url": f"/api/v1/core/tag/{tag.pk}/rename",
+        "delete_url": f"/api/v1/core/tag/{tag.pk}/",
         "snapshots": snapshot_previews or [],
     }
 

archivebox/core/takeover_util.py

@@ -131,10 +131,20 @@ def ensure_daemon_stack(*, reason: str = ""):
             rprint(f"[yellow][*] Starting daemon stack for {reason}...[/yellow]")
         worker = start_worker(supervisor, sonic_worker)
 
-    from archivebox.search.sonic_daemon import wait_for_sonic_daemon
-
-    wait_for_sonic_daemon(sonic_event)
-    return worker
+    # Supervisord marks this worker RUNNING with startsecs=0, before Sonic has
+    # opened its search socket. A query launched at that point fails even though
+    # the daemon is about to become ready.
+    deadline = time.monotonic() + 30.0
+    while time.monotonic() < deadline:
+        if is_port_listening(sonic_event.host, sonic_event.port):
+            if time.monotonic() < deadline:
+                return worker
+            break
+        worker = get_worker(supervisor, sonic_worker["name"])
+        if not isinstance(worker, dict) or worker.get("statename") not in ("STARTING", "RUNNING"):
+            raise RuntimeError(f"Sonic search backend worker failed to start: {worker}")
+        time.sleep(min(0.1, max(0.0, deadline - time.monotonic())))
+    raise RuntimeError(f"Sonic search backend is not listening at {sonic_event.url}")
 
 
 def live_runner_processes(*, data_dir: str | Path):

archivebox/core/templatetags/core_tags.py

@@ -47,6 +47,47 @@ _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml",
 _IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
 _STATIC_URL_SAFE = "/@-._~!$&'()*+,;="
 
+_VIDEO_FILE_EXTS = {
+    ".mp4",
+    ".webm",
+    ".mkv",
+    ".avi",
+    ".mov",
+    ".flv",
+    ".wmv",
+    ".m4v",
+    ".mpg",
+    ".mpeg",
+    ".ts",
+    ".m2ts",
+    ".mts",
+    ".3gp",
+    ".3g2",
+    ".ogv",
+}
+
+_AUDIO_FILE_EXTS = {
+    ".mp3",
+    ".m4a",
+    ".aac",
+    ".ogg",
+    ".oga",
+    ".opus",
+    ".wav",
+    ".flac",
+    ".alac",
+    ".aiff",
+    ".wma",
+    ".mka",
+    ".ac3",
+    ".eac3",
+    ".dts",
+}
+
+_MEDIA_FILE_EXTS = _VIDEO_FILE_EXTS | _AUDIO_FILE_EXTS
+_BROWSER_VIDEO_FILE_EXTS = {".mp4", ".webm", ".m4v", ".ogv"}
+_BROWSER_AUDIO_FILE_EXTS = {".mp3", ".m4a", ".aac", ".ogg", ".oga", ".opus", ".wav", ".flac"}
+
 
 def _is_web_subdomain_alias(browser_url: str, configured_base_url: str) -> bool:
     """Hide the informational mismatch banner on the conventional public web alias.
@@ -63,12 +104,117 @@ def _is_web_subdomain_alias(browser_url: str, configured_base_url: str) -> bool:
     )
 
 
+def _normalize_output_files(output_files: Any) -> dict[str, dict[str, Any]]:
+    from abx_dl.output_files import OutputManifest
+
+    return OutputManifest.from_value(output_files).as_mapping()
+
+
 def _snapshot_id(value: Any) -> Any:
     from archivebox.core.models import Snapshot
 
     return value.id if isinstance(value, Snapshot) else value
 
 
+def _coerce_output_file_size(value: Any) -> int | None:
+    try:
+        return max(int(value or 0), 0)
+    except (TypeError, ValueError):
+        return None
+
+
+def _count_media_files(result) -> int:
+    try:
+        output_files = _normalize_output_files(result.output_files or {})
+    except (AttributeError, TypeError, ValueError):
+        output_files = {}
+
+    if output_files:
+        return sum(1 for path in output_files if Path(path).suffix.lower() in _MEDIA_FILE_EXTS)
+
+    try:
+        plugin_dir = Path(result.snapshot_dir) / result.plugin
+    except (AttributeError, TypeError, ValueError):
+        return 0
+
+    if not plugin_dir.exists():
+        return 0
+
+    count = 0
+    scanned = 0
+    max_scan = 500
+    for file_path in plugin_dir.rglob("*"):
+        if scanned >= max_scan:
+            break
+        scanned += 1
+        if not file_path.is_file():
+            continue
+        if file_path.suffix.lower() in _MEDIA_FILE_EXTS:
+            count += 1
+    return count
+
+
+def _list_media_files(result) -> list[dict]:
+    media_files: list[dict] = []
+    try:
+        plugin_dir = Path(result.snapshot_dir) / result.plugin
+    except (AttributeError, TypeError, ValueError):
+        return media_files
+
+    output_files = _normalize_output_files(result.output_files or {})
+    candidates: list[tuple[Path, int | None]] = []
+    if output_files:
+        for path, metadata in output_files.items():
+            rel_path = Path(path)
+            if rel_path.suffix.lower() in _MEDIA_FILE_EXTS:
+                candidates.append((rel_path, _coerce_output_file_size(metadata.get("size"))))
+
+    if not candidates and plugin_dir.exists():
+        scanned = 0
+        max_scan = 2000
+        for file_path in plugin_dir.rglob("*"):
+            if scanned >= max_scan:
+                break
+            scanned += 1
+            if not file_path.is_file():
+                continue
+            if file_path.suffix.lower() in _MEDIA_FILE_EXTS:
+                try:
+                    rel_path = file_path.relative_to(plugin_dir)
+                except ValueError:
+                    continue
+                try:
+                    size = file_path.stat().st_size
+                except OSError:
+                    size = None
+                candidates.append((rel_path, size))
+
+    for rel_path, size in candidates:
+        href = str(Path(result.plugin) / rel_path)
+        suffix = rel_path.suffix.lower()
+        media_type = "video" if suffix in _VIDEO_FILE_EXTS else "audio"
+        media_files.append(
+            {
+                "name": rel_path.name,
+                "path": href,
+                "size": size,
+                "media_type": media_type,
+                "is_video": media_type == "video",
+                "is_audio": media_type == "audio",
+                "is_browser_playable": suffix in _BROWSER_VIDEO_FILE_EXTS or suffix in _BROWSER_AUDIO_FILE_EXTS,
+            },
+        )
+
+    media_files.sort(
+        key=lambda item: (
+            0 if item["is_video"] and item["is_browser_playable"] else 1 if item["is_audio"] and item["is_browser_playable"] else 2,
+            -int(item.get("size") or 0),
+            item["name"].lower(),
+        ),
+    )
+    return media_files
+
+
 def _resolve_snapshot_output_file(snapshot_dir: str | Path | None, raw_output_path: str | None) -> Path | None:
     if not snapshot_dir or not raw_output_path or str(raw_output_path).strip() in (".", "/", "./"):
         return None
@@ -674,7 +820,8 @@ def plugin_card(context, result) -> str:
 
     icon_html = get_plugin_icon(plugin)
     plugin_lower = (plugin or "").lower()
-    media_files = result.media_files() if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
+    media_file_count = _count_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else 0
+    media_files = _list_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
     if context.get("STATIC_EXPORT") and media_files:
         media_files = [
             item
@@ -684,6 +831,7 @@ def plugin_card(context, result) -> str:
             and ".." not in media_path.parts
             and (Path(result.snapshot_dir) / media_path).is_file()
         ]
+        media_file_count = len(media_files)
     if media_files:
         for item in media_files:
             path = item.get("path") or ""
@@ -704,7 +852,7 @@ def plugin_card(context, result) -> str:
                     "output_path_raw": raw_output_path,
                     "plugin": plugin,
                     "plugin_icon": icon_html,
-                    "media_file_count": len(media_files),
+                    "media_file_count": media_file_count,
                     "media_files": media_files,
                 },
             )

archivebox/core/views/__init__.py

@@ -1,39 +0,0 @@
-"""HTTP views grouped by public browsing, ingestion, replay, and configuration."""
-
-from archivebox.config.common import find_config_source as find_config_source
-from archivebox.plugins.views import get_config_definition_link as get_config_definition_link
-
-from .add import AddView as AddView
-from .add import WebAddView as WebAddView
-from .config import live_config_list_view as live_config_list_view
-from .config import live_config_value_view as live_config_value_view
-from .lookup import _files_index_target as _files_index_target
-from .lookup import _find_snapshot_by_ref as _find_snapshot_by_ref
-from .public import HealthCheckView as HealthCheckView
-from .public import HomepageView as HomepageView
-from .public import PublicIndexView as PublicIndexView
-from .replay import OriginalDomainHostView as OriginalDomainHostView
-from .replay import OriginalDomainReplayView as OriginalDomainReplayView
-from .replay import SnapshotHostView as SnapshotHostView
-from .replay import SnapshotPathView as SnapshotPathView
-from .replay import SnapshotReplayView as SnapshotReplayView
-from .replay import SnapshotView as SnapshotView
-from .replay import _latest_response_match as _latest_response_match
-from .replay import _latest_responses_root as _latest_responses_root
-from .replay import _original_request_url as _original_request_url
-from .replay import _plugin_full_preview_response as _plugin_full_preview_response
-from .replay import _resolve_archiveresult_relpath as _resolve_archiveresult_relpath
-from .replay import _safe_archive_relpath as _safe_archive_relpath
-from .replay import _serve_original_domain_replay as _serve_original_domain_replay
-from .replay import _serve_responses_path as _serve_responses_path
-from .replay import _serve_snapshot_replay as _serve_snapshot_replay
-from .replay import _visible_response_snapshots_for_domain as _visible_response_snapshots_for_domain
-from .replay_auth import REPLAY_AUTH_SALT as REPLAY_AUTH_SALT
-from .replay_auth import SnapshotReplayAuthView as SnapshotReplayAuthView
-from .replay_auth import _admin_login_redirect_or_forbidden as _admin_login_redirect_or_forbidden
-from .replay_auth import _clean_replay_next as _clean_replay_next
-from .replay_auth import _has_replay_cookie as _has_replay_cookie
-from .replay_auth import _private_snapshot_auth_redirect as _private_snapshot_auth_redirect
-from .replay_auth import _replay_auth_response as _replay_auth_response
-from .replay_auth import _replay_cookie_name as _replay_cookie_name
-from .replay_auth import _replay_payload_is_valid as _replay_payload_is_valid

archivebox/core/views/add.py

@@ -1,363 +0,0 @@
-import json
-from urllib.parse import quote
-
-from django.contrib import messages
-from django.contrib.auth.mixins import UserPassesTestMixin
-from django.http import Http404, HttpRequest, HttpResponse, QueryDict
-from django.shortcuts import redirect
-from django.utils.decorators import method_decorator
-from django.utils.html import format_html
-from django.views.decorators.csrf import csrf_exempt, csrf_protect
-from django.views.generic import FormView
-
-from archivebox.config import VERSION
-from archivebox.config.common import (
-    get_config,
-    get_request_config,
-    redact_sensitive_config,
-)
-from archivebox.core.forms import AddLinkForm
-from archivebox.core.models import Snapshot
-from archivebox.core.permissions import (
-    PERMISSIONS_PUBLIC,
-    direct_snapshots_queryset,
-    filter_personas_by_permissions,
-)
-from archivebox.core.routes_util import (
-    build_admin_url,
-    build_web_url,
-    get_admin_host,
-    get_web_host,
-    host_matches,
-)
-from archivebox.crawls.models import Crawl
-from archivebox.misc.util import (
-    sanitize_html_text,
-    urldecode,
-    validate_url,
-)
-from archivebox.plugins.forms import get_plugin_config_binary_urls
-
-from .replay import SnapshotView
-
-
-@method_decorator(csrf_exempt, name="dispatch")
-class AddView(UserPassesTestMixin, FormView):
-    template_name = "add.html"
-    form_class = AddLinkForm
-
-    def get_initial(self):
-        """Prefill the AddLinkForm with the 'url' GET parameter"""
-        if self.request.method == "GET":
-            url = self.request.GET.get("url", None)
-            if url:
-                return {"url": url if "://" in url else f"https://{url}"}
-
-        return super().get_initial()
-
-    def get_form_kwargs(self):
-        kwargs = super().get_form_kwargs()
-        kwargs["request"] = self.request
-        return kwargs
-
-    def test_func(self):
-        return get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated
-
-    def post(self, request: HttpRequest, *args: object, **kwargs: object):
-        if request.user.is_authenticated:
-            return csrf_protect(super().post)(request, *args, **kwargs)
-        return super().post(request, *args, **kwargs)
-
-    def _can_override_crawl_config(self) -> bool:
-        user = self.request.user
-        return bool(user.is_authenticated and user.is_active and user.is_superuser)
-
-    def _get_custom_config_overrides(self, form: AddLinkForm) -> dict:
-        custom_config = form.cleaned_data.get("config") or {}
-
-        if not isinstance(custom_config, dict):
-            return {}
-
-        if not self._can_override_crawl_config():
-            return {}
-
-        return {str(key): value for key, value in custom_config.items() if not str(key).endswith("_BINARY")}
-
-    def get_context_data(self, **kwargs):
-        context = super().get_context_data(**kwargs)
-        request_config = get_request_config(self.request, resolve_plugins=True)
-        required_search_plugin = f"search_backend_{request_config.SEARCH_BACKEND_ENGINE}".strip()
-        can_override_crawl_config = self._can_override_crawl_config()
-        public_persona_config_keys = {
-            "CRAWL_MAX_CONCURRENT_SNAPSHOTS",
-            "DELETE_AFTER",
-            "ONLY_NEW",
-            "PERMISSIONS",
-            "TIMEOUT",
-        }
-        persona_queryset = context["form"].fields["persona"].queryset
-        if not can_override_crawl_config:
-            persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
-        persona_config_map = {}
-        for persona in persona_queryset.order_by("name"):
-            effective_config = get_config(persona=persona)
-            effective_config_redacted = redact_sensitive_config(effective_config.model_dump(mode="json"))
-            if can_override_crawl_config:
-                raw_config = redact_sensitive_config(persona.config or {})
-                effective_config_json = effective_config_redacted
-                binary_urls = get_plugin_config_binary_urls(effective_config)
-            else:
-                raw_config = {}
-                effective_config_json = {key: effective_config_redacted.get(key) for key in public_persona_config_keys}
-                binary_urls = {}
-            persona_config_map[persona.name] = {
-                "config": raw_config,
-                "effective_config": effective_config_json,
-                "binary_urls": binary_urls,
-            }
-        recent_personas = list(persona_queryset.order_by("-created_at", "name")[:5])
-        return {
-            **context,
-            "title": "Create Crawl",
-            # We can't just call request.build_absolute_uri in the template, because it would include query parameters
-            "absolute_add_path": self.request.build_absolute_uri(self.request.path),
-            "web_base_url": build_web_url("", request=self.request),
-            "VERSION": VERSION,
-            "FOOTER_INFO": request_config.FOOTER_INFO,
-            "required_search_plugin": required_search_plugin,
-            "persona_config_map_json": json.dumps(persona_config_map, sort_keys=True, default=str),
-            "recent_personas": recent_personas,
-            "can_override_crawl_config": can_override_crawl_config,
-            "stdout": "",
-        }
-
-    def _create_crawl_from_form(self, form, *, created_by_id=None) -> Crawl:
-        from archivebox.cli.archivebox_add import add
-
-        urls_input = form.cleaned_data["url"]
-        urls = urls_input
-        submitted_lines = [line.strip() for line in urls_input.splitlines() if line.strip()]
-        if len(submitted_lines) == 1:
-            try:
-                # A lone URL pasted into /add/ is the same user-facing input as
-                # `archivebox add https://...`: queue that URL directly so a
-                # narrow plugin selection like `wget` can archive it without
-                # also needing parser plugins. Multi-line or formatted text
-                # remains verbatim import content for the internal parser root.
-                urls = [validate_url(submitted_lines[0])]
-            except ValueError:
-                pass
-        print(f"[+] Adding URL: {urls_input}")
-
-        # Extract all form fields
-        tag = form.cleaned_data["tag"]
-        depth = int(form.cleaned_data["depth"])
-        max_urls = int(form.cleaned_data.get("max_urls") or 0)
-        crawl_max_size = int(form.cleaned_data.get("crawl_max_size") or 0)
-        crawl_timeout = int(form.cleaned_data.get("crawl_timeout") or 0)
-        timeout = form.cleaned_data.get("timeout")
-        snapshot_max_size = int(form.cleaned_data.get("snapshot_max_size") or 0)
-        delete_after = str(form.cleaned_data.get("delete_after") or "0").strip() or "0"
-        crawl_max_concurrent_snapshots = int(form.cleaned_data["crawl_max_concurrent_snapshots"])
-        permissions = str(form.cleaned_data.get("permissions") or "public").strip().lower()
-        can_override_crawl_config = self._can_override_crawl_config()
-        plugins = ",".join(form.cleaned_data.get("plugins", [])) if can_override_crawl_config else ""
-        schedule = form.cleaned_data.get("schedule", "").strip() if can_override_crawl_config else ""
-        persona = form.cleaned_data.get("persona")
-        start_paused = form.cleaned_data.get("start_paused", False) if can_override_crawl_config else False
-        notes = form.cleaned_data.get("notes", "")
-        url_filters = form.cleaned_data.get("url_filters") or {}
-        plugin_config = form.cleaned_data.get("plugin_config") or {}
-        if not isinstance(plugin_config, dict):
-            plugin_config = {}
-        if not can_override_crawl_config:
-            plugin_config = {}
-        custom_config = self._get_custom_config_overrides(form)
-        custom_config.pop("DEFAULT_PERSONA", None)
-        custom_config.pop("PERMISSIONS", None)
-        if persona:
-            persona.ensure_dirs()
-
-        if created_by_id is None:
-            if self.request.user.is_authenticated:
-                created_by_id = self.request.user.pk
-            else:
-                from archivebox.base_models.models import get_or_create_system_user_pk
-
-                created_by_id = get_or_create_system_user_pk()
-
-        config = {}
-        effective_config = get_config(persona=persona) if persona else get_config()
-        if delete_after != str(effective_config.DELETE_AFTER):
-            config["DELETE_AFTER"] = delete_after
-        if timeout is not None and int(timeout) != int(effective_config.TIMEOUT):
-            config["TIMEOUT"] = int(timeout)
-        if permissions:
-            config["PERMISSIONS"] = permissions
-
-        config.update(plugin_config)
-        config.update(custom_config)
-        if bool(url_filters.get("only_new")) != bool(effective_config.ONLY_NEW):
-            config["ONLY_NEW"] = bool(url_filters.get("only_new"))
-        crawl, _snapshots = add(
-            urls=urls,
-            depth=depth,
-            max_urls=max_urls,
-            crawl_max_size=crawl_max_size,
-            crawl_timeout=crawl_timeout,
-            snapshot_max_size=snapshot_max_size,
-            crawl_max_concurrent_snapshots=crawl_max_concurrent_snapshots,
-            tag=tag,
-            url_allowlist=url_filters.get("allowlist") or "",
-            url_denylist=url_filters.get("denylist") or "",
-            plugins=plugins,
-            persona=persona.name if persona else "Default",
-            bg=True,
-            created_by_id=created_by_id,
-            config=config,
-        )
-        if notes:
-            crawl.safe_update({"notes": sanitize_html_text(notes)}, refresh=False)
-        if permissions and crawl.config.get("PERMISSIONS") != permissions:
-            next_config = {**crawl.config, "PERMISSIONS": permissions}
-            crawl.safe_update({"config": next_config}, refresh=True)
-        if start_paused:
-            crawl.pause()
-
-        # 3. create a CrawlSchedule if schedule is provided
-        if schedule:
-            from archivebox.crawls.models import CrawlSchedule
-
-            crawl_schedule = CrawlSchedule.objects.create(
-                template=crawl,
-                schedule=schedule,
-                is_enabled=True,
-                config=config,
-                label=crawl.label,
-                notes=f"Auto-created from add page. {notes}".strip(),
-                created_by_id=created_by_id,
-            )
-            crawl.schedule = crawl_schedule
-            crawl.safe_update({"schedule": crawl_schedule}, refresh=False)
-
-        return crawl
-
-    def form_valid(self, form):
-        crawl = self._create_crawl_from_form(form)
-
-        urls = form.cleaned_data["url"]
-        schedule = form.cleaned_data.get("schedule", "").strip()
-        rough_url_count = len([url for url in urls.splitlines() if url.strip()])
-
-        schedule_msg = ""
-        if schedule and crawl.schedule_id:
-            schedule_msg = format_html(" and <a href='{}'>scheduled to repeat {}</a>", crawl.schedule.admin_change_url, schedule)
-
-        messages.success(
-            self.request,
-            format_html(
-                "Created crawl with {} starting URL(s){}. Snapshots will be created and archived in the background. <a href='{}'>View Crawl →</a>",
-                rough_url_count,
-                schedule_msg,
-                crawl.admin_change_url,
-            ),
-        )
-
-        # Orchestrator (managed by supervisord) will pick up the queued crawl
-        return redirect(crawl.admin_change_url)
-
-
-class WebAddView(AddView):
-    def _latest_snapshot_for_url(self, requested_url: str):
-        return (
-            direct_snapshots_queryset(
-                self.request,
-                SnapshotView.find_snapshots_for_url(requested_url),
-            )
-            .order_by("-bookmarked_at", "-created_at", "-timestamp")
-            .first()
-        )
-
-    def _normalize_add_url(self, requested_url: str) -> str:
-        if requested_url.startswith(("http://", "https://")):
-            return requested_url
-        return f"https://{requested_url}"
-
-    def dispatch(self, request, *args, **kwargs):
-        requested_url = urldecode(kwargs.get("url", "") or "")
-        if requested_url:
-            snapshot = self._latest_snapshot_for_url(requested_url)
-            if snapshot:
-                return redirect(f"/{snapshot.url_path}")
-
-        request_host = (request.get_host() or "").lower()
-        request_config = get_request_config(request)
-        web_host = get_web_host(config=request_config, request=request)
-        admin_host = get_admin_host(config=request_config, request=request)
-        is_web_host = host_matches(request_host, web_host)
-        is_admin_host = host_matches(request_host, admin_host)
-        if request.user.is_authenticated and not request_config.PUBLIC_ADD_VIEW and is_web_host and not is_admin_host:
-            return redirect(build_admin_url(request.get_full_path(), request=request))
-
-        if not self.test_func():
-            if is_web_host and not is_admin_host:
-                return redirect(build_admin_url(request.get_full_path(), request=request))
-            if is_admin_host:
-                next_url = quote(request.get_full_path(), safe="/:?=&")
-                return redirect(f"{build_admin_url('/admin/login/', request=request)}?next={next_url}")
-            return HttpResponse(
-                format_html(
-                    (
-                        "<center><br/><br/><br/>"
-                        "No Snapshots match the given url: <code>{}</code><br/><br/><br/>"
-                        'Return to the <a href="/" target="_top">Main Index</a>'
-                        "</center>"
-                    ),
-                    requested_url or "",
-                ),
-                content_type="text/html",
-                status=404,
-            )
-
-        return super().dispatch(request, *args, **kwargs)
-
-    def get(self, request: HttpRequest, *args: object, **kwargs: object):
-        requested_url = urldecode(str(kwargs.get("url") or (args[0] if args else "")))
-        if not requested_url:
-            raise Http404
-
-        snapshot = self._latest_snapshot_for_url(requested_url)
-        if snapshot:
-            return redirect(f"/{snapshot.url_path}")
-
-        add_url = self._normalize_add_url(requested_url)
-        assert self.form_class is not None
-        defaults_form = self.form_class()
-        form_data = QueryDict(mutable=True)
-        form_data.update(
-            {
-                "url": add_url,
-                "depth": defaults_form.fields["depth"].initial or "0",
-                "max_urls": defaults_form.fields["max_urls"].initial or 0,
-                "crawl_max_size": defaults_form.fields["crawl_max_size"].initial or "0",
-                "crawl_timeout": defaults_form.fields["crawl_timeout"].initial or 0,
-                "timeout": defaults_form.fields["timeout"].initial or 0,
-                "snapshot_max_size": defaults_form.fields["snapshot_max_size"].initial or "0",
-                "delete_after": defaults_form.fields["delete_after"].initial or "0",
-                "crawl_max_concurrent_snapshots": defaults_form.fields["crawl_max_concurrent_snapshots"].initial,
-                "persona": defaults_form.fields["persona"].initial or "Default",
-                "permissions": defaults_form.fields["permissions"].initial or "public",
-                "config": "{}",
-            },
-        )
-        if defaults_form.fields["start_paused"].initial:
-            form_data["start_paused"] = "on"
-
-        form = self.form_class(data=form_data)
-        if not form.is_valid():
-            return self.form_invalid(form)
-
-        crawl = self._create_crawl_from_form(form)
-        snapshot = Snapshot.from_json({"url": add_url, "tags": form.cleaned_data.get("tag", "")}, overrides={"crawl": crawl})
-        assert snapshot is not None
-        return redirect(f"/{snapshot.url_path}")

archivebox/core/views/config.py

@@ -1,261 +0,0 @@
-import os
-from typing import cast
-
-from admin_data_views.typing import ItemContext, SectionData, TableContext
-from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view
-from django.http import HttpRequest
-from django.utils.html import format_html, format_html_join
-from django.utils.safestring import mark_safe
-
-from archivebox.config import CONSTANTS, CONSTANTS_CONFIG
-from archivebox.config.common import (
-    PLUGIN_CONFIG_SCHEMAS,
-    SENSITIVE_CONFIG_VALUE_REDACTED,
-    _plugin_config_properties,
-    find_config_default,
-    find_config_section,
-    find_config_source,
-    find_config_type,
-    get_all_configs,
-    get_config,
-    redact_sensitive_config,
-)
-from archivebox.config.configset import BaseConfigSet
-from archivebox.plugins.views import get_config_definition_link
-
-
-@render_with_table_view
-def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    CONFIGS = get_all_configs()
-
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
-
-    merged_config = get_config(redact_sensitive=True)
-
-    rows = {
-        "Section": [],
-        "Key": [],
-        "Type": [],
-        "Value": [],
-        "Source": [],
-        "Default": [],
-        # "Documentation": [],
-        # "Aliases": [],
-    }
-
-    for section_id, section in reversed(list(CONFIGS.items())):
-        for key in dict(section):
-            rows["Section"].append(section_id)  # section.replace('_', ' ').title().replace(' Config', '')
-            rows["Key"].append(ItemLink(key, key=key))
-            rows["Type"].append(format_html("<code>{}</code>", find_config_type(key)))
-
-            # Use merged config value (includes machine overrides)
-            actual_value = merged_config.get(key, dict(section)[key])
-            rows["Value"].append(format_html("<code>{}</code>", actual_value))
-
-            # Show where the value comes from
-            source = find_config_source(key, merged_config)
-            source_colors = {"Machine": "purple", "Environment": "blue", "File": "green", "Plugin Default": "teal", "Default": "gray"}
-            rows["Source"].append(format_html('<code style="color: {}">{}</code>', source_colors.get(source, "gray"), source))
-
-            rows["Default"].append(
-                format_html(
-                    '<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{}&type=code"><code style="text-decoration: underline">{}</code></a>',
-                    key,
-                    find_config_default(key) or "See here...",
-                ),
-            )
-
-    section = "CONSTANT"
-    for key in CONSTANTS_CONFIG:
-        rows["Section"].append(section)  # section.replace('_', ' ').title().replace(' Config', '')
-        rows["Key"].append(ItemLink(key, key=key))
-        rows["Type"].append(format_html("<code>{}</code>", type(CONSTANTS_CONFIG[key]).__name__))
-        rows["Value"].append(format_html("<code>{}</code>", redact_sensitive_config(CONSTANTS_CONFIG).get(key)))
-        rows["Source"].append(mark_safe('<code style="color: gray">Constant</code>'))
-        rows["Default"].append(
-            format_html(
-                '<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{}&type=code"><code style="text-decoration: underline">{}</code></a>',
-                key,
-                find_config_default(key) or "See here...",
-            ),
-        )
-
-    return TableContext(
-        title="Computed Configuration Values",
-        table=rows,
-    )
-
-
-@render_with_item_view
-def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-    from archivebox.machine.models import Machine
-
-    CONFIGS = get_all_configs()
-
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
-
-    merged_config = get_config(redact_sensitive=True)
-
-    # Determine all sources for this config value
-    sources_info = []
-
-    # Machine config
-    machine = Machine.current()
-    machine_admin_url = machine.admin_change_url
-    if machine.config and key in machine.config:
-        sources_info.append(("Machine", redact_sensitive_config(machine.config).get(key), "purple"))
-
-    # Environment variable
-    if key in os.environ:
-        sources_info.append(("Environment", redact_sensitive_config(os.environ).get(key), "blue"))
-
-    # Config file value
-    if CONSTANTS.CONFIG_FILE.exists():
-        file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
-        if key in file_config:
-            sources_info.append(("File", redact_sensitive_config(file_config).get(key), "green"))
-
-    # Default value
-    default_val = find_config_default(key)
-    if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
-        sources_info.append(("Plugin Default", default_val, "gray"))
-    elif default_val:
-        sources_info.append(("Default", default_val, "gray"))
-
-    # Final computed value
-    config_source = find_config_source(key, merged_config)
-    final_value = merged_config.get(key, CONFIGS.get(key, None))
-    is_redacted = final_value == SENSITIVE_CONFIG_VALUE_REDACTED
-
-    # Build sources display
-    sources_html = format_html_join(
-        mark_safe("<br/>"),
-        '<b style="color: {}">{}:</b> <code>{}</code>',
-        ((color, source, value) for source, value, color in sources_info),
-    )
-
-    aliases = []
-
-    if key in CONSTANTS_CONFIG:
-        section_header = format_html(
-            '[CONSTANTS]   &nbsp; <b><code style="color: lightgray">{}</code></b> &nbsp; <small>(read-only, hardcoded by ArchiveBox)</small>',
-            key,
-        )
-    elif key in merged_config:
-        section_header = format_html(
-            'data / ArchiveBox.conf &nbsp; [{}]  &nbsp; <b><code style="color: lightgray">{}</code></b>',
-            find_config_section(key),
-            key,
-        )
-    else:
-        section_header = format_html(
-            '[DYNAMIC CONFIG]   &nbsp; <b><code style="color: lightgray">{}</code></b> &nbsp; <small>(read-only, calculated at runtime)</small>',
-            key,
-        )
-
-    definition_url, definition_label = get_config_definition_link(key)
-    redacted_message = (
-        mark_safe(
-            '<b style="color: red">Value is redacted for your security. (Passwords, secrets, API tokens, etc. cannot be viewed in the Web UI)</b><br/><br/>',
-        )
-        if is_redacted
-        else ""
-    )
-    default_command_value = val.strip("'") if (val := find_config_default(key)) else str(final_value).strip("'")
-    machine_config_link = (
-        format_html('<br/><a href="{}">→ Edit <code>{}</code> in Machine.config for this server</a>', machine_admin_url, key)
-        if machine_admin_url
-        else ""
-    )
-    machine_config_tip = (
-        format_html(
-            '<br/><b>Tip:</b> To override <code>{}</code> on this machine, <a href="{}">edit the Machine.config field</a> and add:<br/><code>{}</code>',
-            key,
-            machine_admin_url,
-            f'{{"{key}": "your_value_here"}}',
-        )
-        if machine_admin_url and key not in CONSTANTS_CONFIG
-        else ""
-    )
-
-    section_data = cast(
-        SectionData,
-        {
-            "name": section_header,
-            "description": None,
-            "fields": {
-                "Key": key,
-                "Type": find_config_type(key),
-                "Value": final_value,
-                "Currently read from": config_source,
-            },
-            "help_texts": {
-                "Key": format_html(
-                    """
-                <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{}">Documentation</a>  &nbsp;
-                <span style="display: {}">
-                    Aliases: {}
-                </span>
-            """,
-                    key.lower(),
-                    "inline" if aliases else "none",
-                    ", ".join(aliases),
-                ),
-                "Type": format_html(
-                    """
-                <a href="{}" target="_blank" rel="noopener noreferrer">
-                    See full definition in <code>{}</code>...
-                </a>
-            """,
-                    definition_url,
-                    definition_label,
-                ),
-                "Value": format_html(
-                    """
-                {}
-                <br/><hr/><br/>
-                <b>Configuration Sources (highest priority first):</b><br/><br/>
-                {}
-                <br/><br/>
-                <p style="display: {}">
-                    <i>To change this value, edit <code>data/ArchiveBox.conf</code> or run:</i>
-                    <br/><br/>
-                    <code>archivebox config --set {}="{}"</code>
-                </p>
-            """,
-                    redacted_message,
-                    sources_html,
-                    "block" if key in merged_config and key not in CONSTANTS_CONFIG else "none",
-                    key,
-                    default_command_value,
-                ),
-                "Currently read from": format_html(
-                    """
-                The value shown in the "Value" field comes from the <b>{}</b> source.
-                <br/><br/>
-                Priority order (highest to lowest):
-                <ol>
-                    <li><b style="color: purple">Machine</b> - Machine-specific overrides
-                        {}
-                    </li>
-                    <li><b style="color: blue">Environment</b> - process defaults from environment variables</li>
-                    <li><b style="color: green">File</b> - data/ArchiveBox.conf</li>
-                    <li><b style="color: gray">Plugin Default</b> - Default value from plugin config.json</li>
-                    <li><b style="color: gray">Default</b> - Default value from code</li>
-                </ol>
-                {}
-            """,
-                    config_source,
-                    machine_config_link,
-                    machine_config_tip,
-                ),
-            },
-        },
-    )
-
-    return ItemContext(
-        slug=key,
-        title=key,
-        data=[section_data],
-    )

archivebox/core/views/lookup.py

@@ -1,39 +0,0 @@
-from pathlib import Path
-
-from archivebox.core.models import Snapshot
-from archivebox.core.routes_util import (
-    get_snapshot_lookup_key,
-)
-
-
-def _files_index_target(snapshot: Snapshot, archivefile: str | None) -> str:
-    target = archivefile or ""
-    if target == "index.html":
-        target = ""
-    fullpath = Path(snapshot.output_dir) / target
-    if fullpath.is_file():
-        target = str(Path(target).parent)
-        if target == ".":
-            target = ""
-    return target
-
-
-def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
-    lookup = get_snapshot_lookup_key(snapshot_ref)
-    if not lookup:
-        return None
-
-    snapshots = Snapshot.objects.select_related("crawl", "crawl__created_by")
-
-    if len(lookup) == 12 and "-" not in lookup:
-        return snapshots.filter(id__endswith=lookup).order_by("-created_at", "-downloaded_at").first()
-
-    try:
-        return snapshots.get(pk=lookup)
-    except Snapshot.DoesNotExist:
-        try:
-            return snapshots.get(id__startswith=lookup)
-        except Snapshot.DoesNotExist:
-            return None
-        except Snapshot.MultipleObjectsReturned:
-            return snapshots.filter(id__startswith=lookup).first()

archivebox/core/views/public.py

@@ -1,292 +0,0 @@
-from typing import ClassVar
-
-from django.core.paginator import InvalidPage
-from django.db.models import Case, IntegerField, Value, When
-from django.http import Http404, HttpResponse
-from django.shortcuts import redirect
-from django.views import View
-from django.views.generic.list import ListView
-
-from archivebox.config import VERSION
-from archivebox.config.common import (
-    get_request_config,
-)
-from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
-from archivebox.core.permissions import (
-    PERMISSIONS_PRIVATE,
-    PERMISSIONS_PUBLIC,
-    PERMISSIONS_UNLISTED,
-    public_snapshots_queryset,
-)
-from archivebox.core.routes_util import (
-    build_web_url,
-)
-from archivebox.misc.paginators import AcceleratedPaginator
-from archivebox.search.config import (
-    get_search_mode,
-    get_search_mode_backend,
-    get_search_mode_base,
-    get_search_mode_options,
-)
-from archivebox.search.views import get_cached_public_search_state
-
-from .replay_auth import _admin_login_redirect_or_forbidden
-
-
-class HomepageView(View):
-    def get(self, request):
-        request_config = get_request_config(request)
-        if not request_config.BASE_URL:
-            return _admin_login_redirect_or_forbidden(request)
-
-        if request.user.is_authenticated and request_config.CONTROL_PLANE_ENABLED:
-            return redirect("/admin/core/snapshot/")
-
-        if request_config.PUBLIC_INDEX:
-            return redirect("/public")
-
-        return _admin_login_redirect_or_forbidden(request)
-
-
-class PublicIndexView(ListView):
-    template_name = "public_index.html"
-    model = Snapshot
-    ordering: ClassVar[list[str]] = ["-bookmarked_at", "-created_at"]
-    paginator_class = AcceleratedPaginator
-    public_page_scan_chunk_size = 50
-
-    def get_paginate_by(self, queryset):
-        runtime_config = self.__dict__.get("runtime_config")
-        if runtime_config is None:
-            self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
-        return runtime_config.SNAPSHOTS_PER_PAGE
-
-    def _base_public_snapshot_fields(self) -> tuple[str, ...]:
-        return (
-            "id",
-            "created_at",
-            "modified_at",
-            "url",
-            "timestamp",
-            "bookmarked_at",
-            "title",
-            "downloaded_at",
-            "status",
-            "output_size",
-            "permissions",
-        )
-
-    def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot] | None:
-        target_count = page_number * page_size
-        public_snapshots: list[Snapshot] = []
-        scanned = 0
-        chunk_size = max(self.public_page_scan_chunk_size, page_size)
-        ordered_snapshots = Snapshot.objects.order_by(*self.ordering).only(*self._base_public_snapshot_fields())
-
-        while len(public_snapshots) < target_count:
-            chunk = list(ordered_snapshots[scanned : scanned + chunk_size])
-            if not chunk:
-                break
-            scanned += len(chunk)
-            public_snapshots.extend(snapshot for snapshot in chunk if snapshot.permissions == PERMISSIONS_PUBLIC)
-
-        start = (page_number - 1) * page_size
-        return public_snapshots[start:target_count]
-
-    def paginate_queryset(self, queryset, page_size):
-        if self.request.GET.get("q", default="").strip():
-            return super().paginate_queryset(queryset, page_size)
-
-        public_count = self.get_exact_public_snapshot_count()
-        paginator = self.get_paginator(range(public_count), page_size)
-        page_kwarg = self.kwargs.get(self.page_kwarg)
-        page_query = self.request.GET.get(self.page_kwarg)
-        page_number = page_kwarg or page_query or 1
-
-        try:
-            page = paginator.page(page_number)
-        except InvalidPage as err:
-            raise Http404(f"Invalid page ({page_number}): {err}") from err
-
-        object_list = self._ordered_public_page_from_order_index(page_number=page.number, page_size=page_size)
-        page.object_list = object_list
-        return paginator, page, object_list, page.has_other_pages()
-
-    def get_context_data(self, **kwargs):
-        runtime_config = self.__dict__.get("runtime_config")
-        if runtime_config is None:
-            self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
-        search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
-        search_mode_backend = get_search_mode_backend(search_mode, config=runtime_config)
-        query = self.request.GET.get("q", default="").strip()
-        public_search_state = self.__dict__.get("public_search_state")
-        public_search_pending = bool(query and (public_search_state is None or not public_search_state.get("done")))
-        context = {
-            **super().get_context_data(**kwargs),
-            "VERSION": VERSION,
-            "CONFIG": runtime_config,
-            "COMMIT_HASH": runtime_config.COMMIT_HASH,
-            "FOOTER_INFO": runtime_config.FOOTER_INFO,
-            "WEB_BASE_URL": build_web_url(request=self.request, config=runtime_config),
-            "search_mode": search_mode,
-            "search_mode_options": get_search_mode_options(config=runtime_config),
-            "public_search_stream_pending": public_search_pending,
-        }
-        context["show_search_index_hint"] = bool(
-            query
-            and not public_search_pending
-            and get_search_mode_base(search_mode, config=runtime_config) == "deep"
-            and search_mode_backend
-            and context["paginator"].count == 0,
-        )
-        snapshots = list(context.get("object_list") or ())
-        icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots}
-        tag_names_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
-        preview_paths_by_snapshot: dict[str, list[tuple[int, int, str]]] = {str(snapshot.id): [] for snapshot in snapshots}
-        favicon_paths_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
-        progress_by_snapshot: dict[str, dict[str, int]] = {
-            str(snapshot.id): {
-                "total": 0,
-                "succeeded": 0,
-                "failed": 0,
-                "running": 0,
-                "skipped": 0,
-                "noresults": 0,
-            }
-            for snapshot in snapshots
-        }
-        if icons_by_snapshot:
-            for snapshot_id, tag_name in (
-                SnapshotTag.objects.filter(snapshot_id__in=icons_by_snapshot.keys())
-                .order_by("tag__name")
-                .values_list("snapshot_id", "tag__name")
-                .iterator(chunk_size=1000)
-            ):
-                tag_names_by_snapshot[str(snapshot_id)].append(tag_name)
-
-            preview_plugin_order = {
-                "screenshot": 0,
-                "chrome_extension_screenshot": 1,
-            }
-            preview_candidates = {
-                "screenshot": ("screenshot.png",),
-                "chrome_extension_screenshot": ("screenshot-1.png", "screenshot.png"),
-            }
-
-            def result_output_path(result: ArchiveResult, filename: str) -> str | None:
-                output_files = result.output_files or {}
-                file_info = output_files.get(filename)
-                if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
-                    return None
-                if file_info.get("root_relative"):
-                    return filename
-                return f"{result.plugin}/{filename}"
-
-            for snapshot_id, plugin, status in (
-                ArchiveResult.objects.filter(
-                    snapshot_id__in=icons_by_snapshot.keys(),
-                )
-                .exclude(plugin="")
-                .values_list("snapshot_id", "plugin", "status")
-                .iterator(chunk_size=1000)
-            ):
-                snapshot_key = str(snapshot_id)
-                progress = progress_by_snapshot[snapshot_key]
-                progress["total"] += 1
-                if status == ArchiveResult.StatusChoices.SUCCEEDED:
-                    icons_by_snapshot[snapshot_key].add(plugin)
-                    progress["succeeded"] += 1
-                elif status == ArchiveResult.StatusChoices.FAILED:
-                    progress["failed"] += 1
-                elif status == ArchiveResult.StatusChoices.STARTED:
-                    progress["running"] += 1
-                elif status == ArchiveResult.StatusChoices.SKIPPED:
-                    progress["skipped"] += 1
-                elif status == ArchiveResult.StatusChoices.NORESULTS:
-                    progress["noresults"] += 1
-
-            for result in (
-                ArchiveResult.objects.filter(
-                    snapshot_id__in=icons_by_snapshot.keys(),
-                    status=ArchiveResult.StatusChoices.SUCCEEDED,
-                    plugin__in=(*preview_candidates, "favicon"),
-                )
-                .only("snapshot_id", "plugin", "output_files")
-                .iterator(chunk_size=1000)
-            ):
-                snapshot_key = str(result.snapshot_id)
-                if result.plugin in preview_candidates:
-                    plugin_rank = preview_plugin_order[result.plugin]
-                    for filename_rank, filename in enumerate(preview_candidates[result.plugin]):
-                        output_path = result_output_path(result, filename)
-                        if output_path:
-                            preview_paths_by_snapshot[snapshot_key].append((plugin_rank, filename_rank, output_path))
-                elif result.plugin == "favicon":
-                    output_path = result_output_path(result, "favicon.ico")
-                    if output_path:
-                        favicon_paths_by_snapshot[snapshot_key].append(output_path)
-
-        for snapshot in snapshots:
-            snapshot._icons_compact = True
-            snapshot._icons_archive_results = icons_by_snapshot.get(str(snapshot.id), set())
-            snapshot._icons_progress_stats = progress_by_snapshot.get(str(snapshot.id), {})
-            snapshot.num_outputs_cached = snapshot._icons_progress_stats.get("succeeded", 0)
-            snapshot._tags_str_cached = ",".join(tag_names_by_snapshot.get(str(snapshot.id), []))
-            snapshot._public_preview_paths = [
-                output_path for _plugin_rank, _filename_rank, output_path in sorted(preview_paths_by_snapshot.get(str(snapshot.id), []))
-            ]
-            snapshot._public_favicon_paths = favicon_paths_by_snapshot.get(str(snapshot.id), [])
-            snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
-        context["object_list"] = snapshots
-        return context
-
-    def get_exact_public_snapshot_count(self) -> int:
-        hidden_count = Snapshot.objects.filter(permissions=PERMISSIONS_PRIVATE).count()
-        hidden_count += Snapshot.objects.filter(permissions=PERMISSIONS_UNLISTED).count()
-        return Snapshot.objects.count() - hidden_count
-
-    def get_queryset(self, **kwargs):
-        qs = public_snapshots_queryset(super().get_queryset(**kwargs)).only(*self._base_public_snapshot_fields())
-        query = self.request.GET.get("q", default="").strip()
-
-        if not query:
-            return qs
-
-        cached_state = get_cached_public_search_state(self.request)
-        self.public_search_state = cached_state
-        if cached_state is not None:
-            cached_ids = cached_state.get("ids") or []
-            if not cached_ids:
-                return qs.none()
-            search_rank = Case(
-                *(When(pk=snapshot_id, then=Value(index)) for index, snapshot_id in enumerate(cached_ids)),
-                output_field=IntegerField(),
-            )
-            return qs.filter(pk__in=cached_ids).annotate(search_rank=search_rank).order_by("search_rank", *self.ordering)
-
-        return qs.none()
-
-    def get(self, *args, **kwargs):
-        if self.request.user.is_authenticated:
-            return redirect("/admin/core/snapshot/")
-        if get_request_config(self.request).PUBLIC_INDEX:
-            response = super().get(*args, **kwargs)
-            return response
-        else:
-            return _admin_login_redirect_or_forbidden(self.request)
-
-
-class HealthCheckView(View):
-    """
-    A Django view that renders plain text "OK" for service discovery tools
-    """
-
-    def get(self, request):
-        """
-        Handle a GET request
-        """
-        response = HttpResponse("OK", content_type="text/plain", status=200)
-        response["Access-Control-Allow-Origin"] = "*"
-        response["Access-Control-Expose-Headers"] = "X-ArchiveBox-Health"
-        response["X-ArchiveBox-Health"] = "OK"
-        return response

archivebox/core/views/replay.py

@@ -1,832 +0,0 @@
-import posixpath
-from pathlib import Path
-from urllib.parse import quote, urlparse
-
-from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay
-from django import template
-from django.db.models import Q
-from django.http import Http404, HttpRequest, HttpResponse
-from django.shortcuts import redirect, render
-from django.utils.html import format_html
-from django.utils.safestring import mark_safe
-from django.views import View
-
-from archivebox.config.common import (
-    get_request_config,
-)
-from archivebox.core.models import ArchiveResult, Snapshot
-from archivebox.core.permissions import (
-    can_view_snapshot,
-    direct_snapshots_queryset,
-)
-from archivebox.core.routes_util import (
-    build_snapshot_url,
-    build_web_url,
-    get_snapshot_host,
-    host_matches,
-)
-from archivebox.misc.serve_static import serve_static_with_byterange_support
-from archivebox.misc.util import (
-    base_url,
-    filter_queryset_by_uuid_substring,
-    without_fragment,
-)
-from archivebox.plugins.discovery import get_plugin_name, get_plugin_template
-from archivebox.progressmonitor.views import live_progress_view
-
-from .lookup import _files_index_target, _find_snapshot_by_ref
-from .replay_auth import _has_replay_cookie, _private_snapshot_auth_redirect, _replay_auth_response
-
-
-class SnapshotView(View):
-    # render static html index from filesystem archive/<timestamp>/index.html
-
-    @staticmethod
-    def find_snapshots_for_url(path: str, *, allow_fallback: bool = True):
-        """Return a queryset of snapshots matching a URL-ish path. URL only — never tries ID matching.
-
-        Use ``find_snapshots_for_id`` separately if you also want to match by snapshot UUID.
-        """
-
-        def _fragmentless_url_query(url: str) -> Q:
-            from archivebox.misc.db import is_postgres
-
-            canonical = without_fragment(url)
-            if not is_postgres():
-                # Use a range comparison (url >= 'canonical#' AND url < 'canonical#\U0010ffff')
-                # instead of LIKE/__startswith — SQLite's case-insensitive LIKE bypasses the
-                # url index and forces a full-table scan over ~1M rows (~250ms). The range
-                # form lets SQLite use a MULTI-INDEX OR and stays under 1ms.
-                return Q(url=canonical) | (Q(url__gte=f"{canonical}#") & Q(url__lt=f"{canonical}#\U0010ffff"))
-            # On postgres the range trick is unsafe: linguistic (ICU/libc) collations
-            # don't compare '#'-suffixed strings bytewise, so the range can miss rows.
-            # startswith compiles to LIKE 'prefix%' with wildcards escaped, which is
-            # correct under any collation and uses the url pattern-ops index.
-            return Q(url=canonical) | Q(url__startswith=f"{canonical}#")
-
-        normalized = without_fragment(path)
-        if path.startswith(("http://", "https://")):
-            # exact url match (indexed) — fastest path
-            qs = Snapshot.objects.filter(_fragmentless_url_query(path))
-            if not allow_fallback or qs.exists():
-                return qs
-            normalized = normalized.split("://", 1)[1]
-
-        # try exact match on full url (without scheme)
-        qs = Snapshot.objects.filter(
-            _fragmentless_url_query("http://" + normalized) | _fragmentless_url_query("https://" + normalized),
-        )
-        if qs.exists():
-            return qs
-
-        # fall back to match on exact base_url
-        base = base_url(normalized)
-        qs = Snapshot.objects.filter(
-            _fragmentless_url_query("http://" + base) | _fragmentless_url_query("https://" + base),
-        )
-        if qs.exists():
-            return qs
-
-        # fall back to matching base_url as prefix
-        return Snapshot.objects.filter(Q(url__startswith="http://" + base) | Q(url__startswith="https://" + base))
-
-    @staticmethod
-    def find_snapshots_for_id(slug: str):
-        """Return a queryset of snapshots matching a (possibly truncated) UUID via prefix or suffix.
-
-        Strips non-hex characters from ``slug`` (so input with or without hyphens both work).
-        Requires at least 8 hex chars — shorter inputs return an empty queryset to avoid
-        scanning the entire snapshots table on too-broad matches.
-        """
-        return filter_queryset_by_uuid_substring(Snapshot.objects.all(), slug)
-
-    @staticmethod
-    def render_live_index(request, snapshot):
-        return render(
-            template_name="core/snapshot.html",
-            request=request,
-            context=snapshot.get_html_details_context(request=request),
-        )
-
-    def get(self, request, path):
-        snapshot = None
-
-        try:
-            slug, archivefile = path.split("/", 1)
-        except (IndexError, ValueError):
-            slug, archivefile = path.split("/", 1)[0], "index.html"
-
-        # slug is a timestamp
-        if slug.replace(".", "").isdigit():
-            # missing trailing slash -> redirect to index
-            if "/" not in path:
-                return redirect(f"{path}/index.html")
-
-            try:
-                try:
-                    snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
-                    if not can_view_snapshot(request, snapshot):
-                        return _private_snapshot_auth_redirect(request, snapshot, archivefile or "index.html")
-                    canonical_base = snapshot.url_path
-                    if canonical_base != snapshot.legacy_archive_path:
-                        target_path = f"/{canonical_base}/{archivefile or 'index.html'}"
-                        query = request.META.get("QUERY_STRING")
-                        if query:
-                            target_path = f"{target_path}?{query}"
-                        return redirect(target_path)
-
-                    if request.GET.get("files"):
-                        target_path = _files_index_target(snapshot, archivefile)
-                        response = serve_static_with_byterange_support(
-                            request,
-                            target_path,
-                            document_root=snapshot.output_dir,
-                            show_indexes=True,
-                            is_archive_replay=True,
-                        )
-                    elif archivefile == "index.html":
-                        # if they requested snapshot index, serve live rendered template instead of static html
-                        response = self.render_live_index(request, snapshot)
-                    else:
-                        target = build_snapshot_url(str(snapshot.id), archivefile, request=request)
-                        query = request.META.get("QUERY_STRING")
-                        if query:
-                            target = f"{target}?{query}"
-                        return redirect(target)
-                    response["Link"] = f'<{snapshot.url}>; rel="canonical"'
-                    return response
-                except Snapshot.DoesNotExist:
-                    if Snapshot.objects.filter(timestamp__startswith=slug).exists():
-                        raise Snapshot.MultipleObjectsReturned
-                    else:
-                        raise
-            except Snapshot.DoesNotExist:
-                # Snapshot does not exist
-                return HttpResponse(
-                    format_html(
-                        (
-                            "<center><br/><br/><br/>"
-                            "No Snapshot directories match the given timestamp/ID: <code>{}</code><br/><br/>"
-                            'You can <a href="/add/" target="_top">add a new Snapshot</a>, or return to the <a href="/" target="_top">Main Index</a>'
-                            "</center>"
-                        ),
-                        slug,
-                        path,
-                    ),
-                    content_type="text/html",
-                    status=404,
-                )
-            except Snapshot.MultipleObjectsReturned:
-                snapshot_hrefs = mark_safe("<br/>").join(
-                    format_html(
-                        '{} <a href="/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
-                        snap.bookmarked_at.strftime("%Y-%m-%d %H:%M:%S"),
-                        snap.archive_path,
-                        snap.timestamp,
-                        snap.url,
-                        snap.title_stripped[:64] or "",
-                    )
-                    for snap in direct_snapshots_queryset(request, Snapshot.objects.filter(timestamp__startswith=slug))
-                    .only("url", "timestamp", "title", "bookmarked_at")
-                    .order_by("-bookmarked_at")
-                )
-                return HttpResponse(
-                    format_html(
-                        ("Multiple Snapshots match the given timestamp/ID <code>{}</code><br/><pre>"),
-                        slug,
-                    )
-                    + snapshot_hrefs
-                    + mark_safe('</pre><br/>Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'),
-                    content_type="text/html",
-                    status=404,
-                )
-            except Http404:
-                assert snapshot  # (Snapshot.DoesNotExist is already handled above)
-
-                # Snapshot dir exists but file within does not e.g. 124235.324234/screenshot.png
-                return HttpResponse(
-                    format_html(
-                        """
-                        <html><head>
-                        <title>Snapshot Not Found</title>
-                        </head><body>
-                        <center><br/><br/><br/>
-                        Snapshot <a href="/{}/index.html" target="_top"><b><code>[{}]</code></b></a>: <a href="{}" target="_blank" rel="noreferrer">{}</a><br/>
-                        was queued on {}, but no files have been saved yet in:<br/><b><a href="/{}/" target="_top"><code>{}</code></a><code>/{}</code></b><br/><br/>
-                        It's possible {} during the last capture on {},<br/>or that the archiving process has not completed yet.<br/>
-                        <pre><code># run this cmd to finish/retry archiving this Snapshot</code><br/>
-                        <code style="user-select: all; color: #333">archivebox update -t timestamp {}</code></pre><br/><br/>
-                        <div class="text-align: left; width: 100%; max-width: 400px">
-                        <i><b>Next steps:</i></b><br/>
-                        - list all the <a href="/{}/" target="_top">Snapshot files <code>.*</code></a><br/>
-                        - view the <a href="/{}/index.html" target="_top">Snapshot <code>./index.html</code></a><br/>
-                        - go to the <a href="/admin/core/snapshot/{}/change/" target="_top">Snapshot admin</a> to edit<br/>
-                        - go to the <a href="/admin/core/snapshot/?id__exact={}" target="_top">Snapshot actions</a> to re-archive<br/>
-                        - or return to <a href="/" target="_top">the main index...</a></div>
-                        </center>
-                        </body></html>
-                        """,
-                        snapshot.archive_path,
-                        snapshot.timestamp,
-                        snapshot.url,
-                        snapshot.url,
-                        str(snapshot.bookmarked_at).split(".")[0],
-                        snapshot.archive_path,
-                        snapshot.timestamp,
-                        archivefile if str(archivefile) != "None" else "",
-                        f"the {archivefile} resource could not be fetched"
-                        if str(archivefile) != "None"
-                        else "the original site was not available",
-                        str(snapshot.bookmarked_at).split(".")[0],
-                        snapshot.timestamp,
-                        snapshot.archive_path,
-                        snapshot.archive_path,
-                        snapshot.pk,
-                        snapshot.id,
-                    ),
-                    content_type="text/html",
-                    status=404,
-                )
-
-        # slug is either a URL or a (possibly truncated) snapshot UUID
-        def _resolve_snapshots_for_slug(slug: str):
-            # full URLs go straight to the url-only path (fast, indexed)
-            if "://" in slug:
-                return SnapshotView.find_snapshots_for_url(slug)
-            # short uuid-shaped slugs (>=8 hex chars after stripping non-hex) try id matching first
-            id_qs = SnapshotView.find_snapshots_for_id(slug)
-            if id_qs.exists():
-                return id_qs
-            return SnapshotView.find_snapshots_for_url(slug)
-
-        snapshots = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path))
-        try:
-            if "://" in path:
-                snapshot = snapshots.order_by("-bookmarked_at").first()
-                if snapshot is None:
-                    raise Snapshot.DoesNotExist
-            else:
-                snapshot = snapshots.get()
-        except Snapshot.DoesNotExist:
-            return HttpResponse(
-                format_html(
-                    (
-                        "<center><br/><br/><br/>"
-                        "No Snapshots match the given url: <code>{}</code><br/><br/><br/>"
-                        'Return to the <a href="/" target="_top">Main Index</a>, or:<br/><br/>'
-                        '+ <i><a href="/add/?url={}" target="_top">Add a new Snapshot for <code>{}</code></a><br/><br/></i>'
-                        "</center>"
-                    ),
-                    base_url(path),
-                    path if "://" in path else f"https://{path}",
-                    path,
-                ),
-                content_type="text/html",
-                status=404,
-            )
-        except Snapshot.MultipleObjectsReturned:
-            snapshot_hrefs = mark_safe("<br/>").join(
-                format_html(
-                    '{} <code style="font-size: 0.8em">{}</code> <a href="/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
-                    snap.bookmarked_at.strftime("%Y-%m-%d %H:%M:%S"),
-                    str(snap.id)[:8],
-                    snap.archive_path,
-                    snap.timestamp,
-                    snap.url,
-                    snap.title_stripped[:64] or "",
-                )
-                for snap in snapshots.only("url", "timestamp", "title", "bookmarked_at").order_by("-bookmarked_at")
-            )
-            return HttpResponse(
-                format_html(
-                    ("Multiple Snapshots match the given URL <code>{}</code><br/><pre>"),
-                    base_url(path),
-                )
-                + snapshot_hrefs
-                + mark_safe('</pre><br/>Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'),
-                content_type="text/html",
-                status=404,
-            )
-
-        target_path = build_snapshot_url(str(snapshot.id), "index.html", request=request)
-        query = request.META.get("QUERY_STRING")
-        if query:
-            target_path = f"{target_path}?{query}"
-        return redirect(target_path)
-
-
-class SnapshotPathView(View):
-    """Serve snapshots by the new URL scheme: /<username>/<YYYYMMDD>/<domain>/<uuid>/..."""
-
-    def get(
-        self,
-        request,
-        username: str,
-        date: str | None = None,
-        domain: str | None = None,
-        snapshot_id: str | None = None,
-        path: str = "",
-        url: str | None = None,
-    ):
-        if username == "system":
-            return redirect(request.path.replace("/system/", "/web/", 1))
-
-        if date and domain and domain == date:
-            raise Http404
-
-        requested_url = url
-        if not requested_url and domain and domain.startswith(("http://", "https://")):
-            requested_url = domain
-
-        snapshot = None
-        snapshots_qs = direct_snapshots_queryset(request, Snapshot.objects.select_related("crawl", "crawl__created_by"))
-        if snapshot_id:
-            snapshot = _find_snapshot_by_ref(snapshot_id)
-            if snapshot and not can_view_snapshot(request, snapshot):
-                return _private_snapshot_auth_redirect(request, snapshot, path or "index.html")
-        else:
-            # fuzzy lookup by date + domain/url (most recent)
-            username_lookup = "system" if username == "web" else username
-            if requested_url:
-                qs = direct_snapshots_queryset(
-                    request,
-                    SnapshotView.find_snapshots_for_url(requested_url)
-                    .select_related("crawl", "crawl__created_by")
-                    .filter(
-                        crawl__created_by__username=username_lookup,
-                    ),
-                )
-            else:
-                qs = snapshots_qs.filter(crawl__created_by__username=username_lookup)
-
-            if date:
-                try:
-                    if len(date) == 4:
-                        qs = qs.filter(bookmarked_at__year=int(date))
-                    elif len(date) == 6:
-                        qs = qs.filter(bookmarked_at__year=int(date[:4]), bookmarked_at__month=int(date[4:6]))
-                    elif len(date) == 8:
-                        qs = qs.filter(
-                            bookmarked_at__year=int(date[:4]),
-                            bookmarked_at__month=int(date[4:6]),
-                            bookmarked_at__day=int(date[6:8]),
-                        )
-                except ValueError:
-                    pass
-
-            if requested_url:
-                snapshot = qs.order_by("-bookmarked_at", "-created_at", "-timestamp").first()
-            else:
-                requested_domain = domain or ""
-                if requested_domain.startswith(("http://", "https://")):
-                    requested_domain = Snapshot.extract_domain_from_url(requested_domain)
-                else:
-                    requested_domain = Snapshot.extract_domain_from_url(f"https://{requested_domain}")
-
-                # Prefer exact domain matches
-                matches = [
-                    s for s in qs.order_by("-bookmarked_at", "-created_at") if Snapshot.extract_domain_from_url(s.url) == requested_domain
-                ]
-                snapshot = matches[0] if matches else qs.order_by("-bookmarked_at", "-created_at", "-timestamp").first()
-
-        if not snapshot:
-            return HttpResponse(
-                format_html(
-                    (
-                        "<center><br/><br/><br/>"
-                        "No Snapshots match the given id or url: <code>{}</code><br/><br/><br/>"
-                        'Return to the <a href="/" target="_top">Main Index</a>'
-                        "</center>"
-                    ),
-                    snapshot_id or requested_url or domain,
-                ),
-                content_type="text/html",
-                status=404,
-            )
-
-        canonical_base = snapshot.url_path
-        if date:
-            requested_base = f"{username}/{date}/{domain or url or ''}"
-        else:
-            requested_base = f"{username}/{domain or url or ''}"
-        if snapshot_id:
-            requested_base = f"{requested_base}/{snapshot_id}"
-        if canonical_base != requested_base:
-            target = f"/{canonical_base}/{path or 'index.html'}"
-            query = request.META.get("QUERY_STRING")
-            if query:
-                target = f"{target}?{query}"
-            return redirect(target)
-
-        archivefile = path or "index.html"
-        if archivefile != "index.html" and not request.GET.get("files"):
-            target = build_snapshot_url(str(snapshot.id), archivefile, request=request)
-            query = request.META.get("QUERY_STRING")
-            if query:
-                target = f"{target}?{query}"
-            return redirect(target)
-
-        if request.GET.get("files"):
-            target_path = _files_index_target(snapshot, archivefile)
-            return serve_static_with_byterange_support(
-                request,
-                target_path,
-                document_root=snapshot.output_dir,
-                show_indexes=True,
-                is_archive_replay=True,
-            )
-
-        if archivefile == "index.html":
-            return SnapshotView.render_live_index(request, snapshot)
-
-        return serve_static_with_byterange_support(
-            request,
-            archivefile,
-            document_root=snapshot.output_dir,
-            show_indexes=True,
-            is_archive_replay=True,
-        )
-
-
-def _safe_archive_relpath(path: str) -> str | None:
-    if not path:
-        return ""
-    cleaned = posixpath.normpath(path)
-    cleaned = cleaned.lstrip("/")
-    if cleaned.startswith("..") or "/../" in f"/{cleaned}/":
-        return None
-    return cleaned
-
-
-def _resolve_archiveresult_relpath(snapshot: Snapshot, rel_path: str) -> tuple[str, ArchiveResult | None]:
-    """Resolve plugin-relative output paths through ArchiveResult.output_files."""
-    parts = Path(rel_path).parts
-    if len(parts) < 2:
-        return rel_path, None
-
-    plugin = parts[0]
-    plugin_relpath = posixpath.join(*parts[1:])
-    results = list(
-        ArchiveResult.objects.filter(
-            snapshot=snapshot,
-            plugin=plugin,
-            status=ArchiveResult.StatusChoices.SUCCEEDED,
-        ).only("plugin", "output_files"),
-    )
-    if not results:
-        return rel_path, None
-
-    for result in results:
-        output_files = result.output_files or {}
-        for candidate in (plugin_relpath, rel_path):
-            file_info = output_files.get(candidate)
-            if not isinstance(file_info, dict):
-                continue
-            if file_info.get("root_relative"):
-                return candidate, result
-            return rel_path, result
-
-    return rel_path, results[0]
-
-
-def _plugin_full_preview_response(
-    request: HttpRequest,
-    snapshot: Snapshot,
-    rel_path: str,
-    result: ArchiveResult | None,
-) -> HttpResponse | None:
-    """Render an explicit plugin full template as a trusted preview wrapper."""
-    if not request.GET.get("preview"):
-        return None
-
-    path_parts = Path(rel_path).parts
-    plugin = get_plugin_name(result.plugin) if result else (path_parts[0] if len(path_parts) > 1 else "")
-    if not plugin:
-        return None
-
-    # ReplayWeb.page needs plugin-owned WACZ inspection and service-worker
-    # context, so it remains the one narrow preview exception below.
-    if plugin == "archivewebpage" and archivewebpage_replay.is_replay_target(rel_path):
-        return None
-
-    template_str = get_plugin_template(plugin, "full", fallback=False)
-    if not template_str:
-        return None
-
-    raw_query = request.GET.copy()
-    raw_query.pop("preview", None)
-    output_url = request.path
-    if raw_query:
-        output_url = f"{output_url}?{raw_query.urlencode()}"
-
-    rendered = (
-        template.Engine(debug=False)
-        .from_string(template_str)
-        .render(
-            template.Context(
-                {
-                    "result": result,
-                    "snapshot": snapshot,
-                    "output_path": output_url,
-                    "output_path_raw": rel_path,
-                    "plugin": plugin,
-                    "preview_base": f"{request.path.rsplit('/', 1)[0]}/",
-                },
-            ),
-        )
-    )
-    response = HttpResponse(rendered, content_type="text/html; charset=utf-8")
-    response.headers["Content-Disposition"] = f'inline; filename="{Path(rel_path).stem}.html"'
-    response.headers["X-Content-Type-Options"] = "nosniff"
-    response.headers["X-ArchiveBox-Security-Mode"] = request.archivebox_config.SERVER_SECURITY_MODE
-    response.headers["Referrer-Policy"] = "no-referrer"
-    response.headers["Content-Security-Policy"] = (
-        "default-src 'self' data: blob:; "
-        "script-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; "
-        "style-src 'unsafe-inline' data: blob: 'self'; "
-        "connect-src 'self' data: blob:; "
-        "img-src 'self' data: blob:; "
-        "media-src 'self' data: blob:; "
-        "font-src 'self' data: blob:; "
-        "frame-src 'self' data: blob:; "
-        "worker-src 'self' blob:; "
-        "object-src 'none'; "
-        "base-uri 'none'; "
-        "form-action 'none'; "
-        "frame-ancestors 'self';"
-    )
-    return response
-
-
-def _visible_response_snapshots_for_domain(request: HttpRequest, domain: str) -> list[Snapshot]:
-    if not domain:
-        return []
-
-    requested_domain = domain.split(":", 1)[0].lower()
-    roots = (f"http://{requested_domain}", f"https://{requested_domain}")
-    domain_query = Q(url__in=roots)
-
-    from archivebox.misc.db import is_postgres
-
-    postgres = is_postgres()
-    for root in roots:
-        for separator in ("/", "?", "#"):
-            prefix = f"{root}{separator}"
-            if postgres:
-                domain_query |= Q(url__startswith=prefix)
-            else:
-                domain_query |= Q(url__gte=prefix, url__lt=f"{prefix}\U0010ffff")
-
-    candidates = (
-        Snapshot.objects.filter(domain_query)
-        .select_related("crawl", "crawl__created_by")
-        .order_by("-bookmarked_at", "-created_at", "-timestamp")
-    )
-    return [snapshot for snapshot in candidates if can_view_snapshot(request, snapshot) or _has_replay_cookie(request, snapshot)]
-
-
-def _latest_response_match(
-    snapshots: list[Snapshot],
-    domain: str,
-    rel_path: str,
-) -> tuple[Snapshot, Path, Path] | None:
-    if not domain or not rel_path:
-        return None
-    requested_domain = domain.split(":", 1)[0].lower()
-    rel_to_root = Path(rel_path)
-    for snapshot in snapshots:
-        responses_root = Path(snapshot.output_dir) / "responses" / requested_domain
-        if (responses_root / rel_to_root).exists():
-            return snapshot, responses_root, rel_to_root
-    return None
-
-
-def _latest_responses_root(snapshots: list[Snapshot], domain: str) -> tuple[Snapshot, Path] | None:
-    if not domain:
-        return None
-    requested_domain = domain.split(":", 1)[0].lower()
-    for snapshot in snapshots:
-        responses_root = Path(snapshot.output_dir) / "responses" / requested_domain
-        if responses_root.is_dir():
-            return snapshot, responses_root
-    return None
-
-
-def _original_request_url(domain: str, path: str = "", query_string: str = "") -> str:
-    normalized_domain = (domain or "").split(":", 1)[0].lower()
-    normalized_path = (path or "").lstrip("/")
-    if normalized_path in ("", "index.html"):
-        normalized_path = ""
-    target = f"https://{normalized_domain}"
-    if normalized_path:
-        target = f"{target}/{normalized_path}"
-    if query_string:
-        target = f"{target}?{query_string}"
-    return target
-
-
-def _serve_responses_path(request, responses_root: Path, rel_path: str, show_indexes: bool):
-    candidates: list[str] = []
-    rel_path = rel_path or ""
-    if rel_path.endswith("/"):
-        rel_path = f"{rel_path}index.html"
-    if "." not in Path(rel_path).name:
-        candidates.append(f"{rel_path.rstrip('/')}/index.html")
-    candidates.append(rel_path)
-
-    for candidate in candidates:
-        try:
-            return serve_static_with_byterange_support(
-                request,
-                candidate,
-                document_root=str(responses_root),
-                show_indexes=show_indexes,
-                is_archive_replay=True,
-            )
-        except Http404:
-            pass
-
-    if rel_path.endswith("index.html"):
-        rel_dir = rel_path[: -len("index.html")]
-        try:
-            return serve_static_with_byterange_support(
-                request,
-                rel_dir,
-                document_root=str(responses_root),
-                show_indexes=True,
-                is_archive_replay=True,
-            )
-        except Http404:
-            return None
-    return None
-
-
-def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""):
-    rel_path = path or ""
-    request_config = get_request_config(
-        request,
-        resolve_plugins=rel_path.startswith("replay/") or rel_path == "replay",
-    )
-    request.archivebox_config = request_config
-    request.archivebox_snapshot_url = snapshot.url
-    snapshot._runtime_config = request_config
-
-    if rel_path.startswith("replay/") or rel_path == "replay":
-        response = archivewebpage_replay.serve_replay_asset_response(rel_path, request_config, HttpResponse)
-        if response is not None:
-            return response
-
-    if rel_path == "progress.json":
-        # Host routing forwards every snap-* path to SnapshotHostView, so we forward
-        # /progress.json on through to the same view used everywhere else. The caller
-        # passes snapshot_id explicitly in the query string — we don't read it from the
-        # subdomain (this keeps the endpoint identical across all security modes).
-        return live_progress_view(request)
-
-    is_directory_request = bool(path) and path.endswith("/")
-    show_indexes = bool(request.GET.get("files")) or (request_config.USES_SUBDOMAIN_ROUTING and is_directory_request)
-    if not show_indexes and (not rel_path or rel_path == "index.html"):
-        return SnapshotView.render_live_index(request, snapshot)
-
-    if not rel_path or rel_path.endswith("/"):
-        if show_indexes:
-            rel_path = rel_path.rstrip("/")
-        else:
-            rel_path = f"{rel_path}index.html"
-    rel_path = _safe_archive_relpath(rel_path)
-    if rel_path is None:
-        raise Http404
-
-    rel_path, archive_result = _resolve_archiveresult_relpath(snapshot, rel_path)
-
-    plugin_preview = _plugin_full_preview_response(request, snapshot, rel_path, archive_result)
-    if plugin_preview is not None:
-        return plugin_preview
-
-    try:
-        return serve_static_with_byterange_support(
-            request,
-            rel_path,
-            document_root=snapshot.output_dir,
-            show_indexes=show_indexes,
-            is_archive_replay=True,
-        )
-    except Http404:
-        pass
-
-    host = urlparse(snapshot.url).hostname or snapshot.domain
-    responses_root = Path(snapshot.output_dir) / "responses" / host
-    if responses_root.exists():
-        response = _serve_responses_path(request, responses_root, rel_path, show_indexes)
-        if response is not None:
-            return response
-
-    raise Http404
-
-
-def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str = ""):
-    request_config = get_request_config(request, resolve_plugins=False)
-    request.archivebox_config = request_config
-    requested_root_index = path in ("", "index.html") or path.endswith("/")
-    rel_path = path or ""
-    if not rel_path or rel_path.endswith("/"):
-        rel_path = f"{rel_path}index.html"
-    rel_path = _safe_archive_relpath(rel_path)
-    if rel_path is None:
-        raise Http404
-
-    domain = domain.split(":", 1)[0].lower()
-    snapshots = _visible_response_snapshots_for_domain(request, domain)
-    match = _latest_response_match(snapshots, domain, rel_path)
-    if not match and "." not in Path(rel_path).name:
-        index_path = f"{rel_path.rstrip('/')}/index.html"
-        match = _latest_response_match(snapshots, domain, index_path)
-    if not match and "." not in Path(rel_path).name:
-        html_path = f"{rel_path}.html"
-        match = _latest_response_match(snapshots, domain, html_path)
-
-    root_match = (match[0], match[1]) if match else _latest_responses_root(snapshots, domain)
-    responses_root = root_match[1] if root_match else None
-    if request_config.USES_SUBDOMAIN_ROUTING:
-        snapshot = root_match[0] if root_match else (snapshots[0] if requested_root_index and snapshots else None)
-        if snapshot:
-            snapshot_path = f"responses/{domain}/{match[2]}" if match else path
-            target = build_snapshot_url(str(snapshot.id), snapshot_path, request=request, config=request_config)
-            if request.META.get("QUERY_STRING"):
-                target = f"{target}?{request.META['QUERY_STRING']}"
-            return redirect(target)
-
-    show_indexes = bool(request.GET.get("files"))
-    if match:
-        _snapshot, responses_root, rel_to_root = match
-        response = _serve_responses_path(request, responses_root, str(rel_to_root), show_indexes)
-        if response is not None:
-            return response
-
-    if responses_root:
-        response = _serve_responses_path(request, responses_root, rel_path, show_indexes)
-        if response is not None:
-            return response
-
-    if requested_root_index and not show_indexes:
-        if snapshots:
-            return SnapshotView.render_live_index(request, snapshots[0])
-
-    if request_config.PUBLIC_ADD_VIEW or request.user.is_authenticated:
-        target_url = _original_request_url(domain, path, request.META.get("QUERY_STRING", ""))
-        return redirect(build_web_url(f"/web/{quote(target_url, safe=':/')}"))
-
-    raise Http404
-
-
-class SnapshotHostView(View):
-    """Serve snapshot directory contents on <snapshot-subdomain>.<listen_host>/<path>."""
-
-    def get(self, request, snapshot_id: str, path: str = ""):
-        request_config = get_request_config(request)
-        snapshot = _find_snapshot_by_ref(snapshot_id)
-
-        if not snapshot:
-            raise Http404
-        if path == "_auth":
-            return _replay_auth_response(request, snapshot)
-        if not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot):
-            return _private_snapshot_auth_redirect(request, snapshot, path)
-
-        canonical_host = get_snapshot_host(str(snapshot.id), config=request_config)
-        if not host_matches(request.get_host(), canonical_host):
-            target = build_snapshot_url(str(snapshot.id), path, request=request, config=request_config)
-            if request.META.get("QUERY_STRING"):
-                target = f"{target}?{request.META['QUERY_STRING']}"
-            return redirect(target)
-
-        return _serve_snapshot_replay(request, snapshot, path)
-
-
-class SnapshotReplayView(View):
-    """Serve snapshot directory contents on a one-domain replay path."""
-
-    def get(self, request, snapshot_id: str, path: str = ""):
-        snapshot = _find_snapshot_by_ref(snapshot_id)
-        if not snapshot:
-            raise Http404
-        if path == "_auth":
-            return _replay_auth_response(request, snapshot)
-        if not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot):
-            return _private_snapshot_auth_redirect(request, snapshot, path)
-
-        return _serve_snapshot_replay(request, snapshot, path)
-
-
-class OriginalDomainHostView(View):
-    """Serve responses from the most recent snapshot when using <domain>.<listen_host>/<path>."""
-
-    def get(self, request, domain: str, path: str = ""):
-        return _serve_original_domain_replay(request, domain, path)
-
-
-class OriginalDomainReplayView(View):
-    """Serve original-domain replay content on a one-domain replay path."""
-
-    def get(self, request, domain: str, path: str = ""):
-        return _serve_original_domain_replay(request, domain, path)

archivebox/core/views/replay_auth.py

@@ -1,148 +0,0 @@
-from urllib.parse import quote, urlparse
-
-from django.conf import settings
-from django.contrib.auth import HASH_SESSION_KEY, SESSION_KEY, get_user_model
-from django.contrib.sessions.models import Session
-from django.core import signing
-from django.http import Http404, HttpRequest, HttpResponseForbidden
-from django.shortcuts import redirect
-from django.views import View
-
-from archivebox.config import CONSTANTS
-from archivebox.config.common import (
-    get_request_config,
-)
-from archivebox.core.models import Snapshot
-from archivebox.core.permissions import (
-    is_admin_user,
-)
-from archivebox.core.routes_util import (
-    build_admin_url,
-    build_snapshot_url,
-)
-
-from .lookup import _find_snapshot_by_ref
-
-REPLAY_AUTH_SALT = "archivebox.private-snapshot-replay"
-REPLAY_COOKIE_PREFIX = f"archivebox_replay_{CONSTANTS.COLLECTION_ID}_"
-REPLAY_GRANT_MAX_AGE = 60
-
-
-def _admin_login_redirect_or_forbidden(request: HttpRequest):
-    if get_request_config(request).CONTROL_PLANE_ENABLED:
-        return redirect(f"/admin/login/?next={request.path}")
-    return HttpResponseForbidden("ArchiveBox is running with the control plane disabled in this security mode.")
-
-
-def _replay_cookie_name(snapshot: Snapshot) -> str:
-    return f"{REPLAY_COOKIE_PREFIX}{str(snapshot.id).replace('-', '')[-12:]}"
-
-
-def _clean_replay_next(path: str | None) -> str:
-    """Only allow same-snap relative replay paths; grants must never redirect off-host."""
-    path = f"/{(path or 'index.html').lstrip('/')}"
-    parsed = urlparse(path)
-    if parsed.scheme or parsed.netloc or path.startswith("//"):
-        return "/index.html"
-    return path
-
-
-def _replay_payload_is_valid(payload: dict, snapshot: Snapshot) -> bool:
-    """A replay cookie is not its own auth source; it must point at a live admin session.
-
-    Replayed pages can execute hostile JS, so admin cookies stay host-only on admin.*.
-    The snap host gets only this host-only HttpOnly cookie, and every request checks
-    that the original Django session still exists and still belongs to an active staff user.
-    Logout, session expiry, user deletion/deactivation, or password auth-hash rotation all
-    make the replay cookie inert without needing admin.* to delete a cookie on snap-*.
-    """
-    if payload.get("snapshot_id") != str(snapshot.id):
-        return False
-    try:
-        session = Session.objects.get(session_key=str(payload.get("session_key") or ""))
-        session_data = session.get_decoded()
-        user_id = str(session_data.get(SESSION_KEY) or "")
-        auth_hash = str(session_data.get(HASH_SESSION_KEY) or "")
-        user = get_user_model().objects.get(pk=user_id)
-    except (Session.DoesNotExist, get_user_model().DoesNotExist, KeyError, TypeError, ValueError):
-        return False
-    return (
-        str(payload.get("user_id")) == user_id
-        and str(payload.get("auth_hash") or "") == auth_hash
-        and user.is_active
-        and user.is_staff
-        and auth_hash == user.get_session_auth_hash()
-    )
-
-
-def _has_replay_cookie(request: HttpRequest, snapshot: Snapshot) -> bool:
-    value = request.COOKIES.get(_replay_cookie_name(snapshot))
-    if not value:
-        return False
-    try:
-        payload = signing.loads(value, salt=REPLAY_AUTH_SALT, max_age=settings.SESSION_COOKIE_AGE)
-    except signing.BadSignature:
-        return False
-    return isinstance(payload, dict) and _replay_payload_is_valid(payload, snapshot)
-
-
-def _private_snapshot_auth_redirect(request: HttpRequest, snapshot: Snapshot, path: str = "", *, preserve_query: bool = True):
-    next_path = _clean_replay_next(path or "index.html")
-    if preserve_query and request.META.get("QUERY_STRING"):
-        next_path = f"{next_path}?{request.META['QUERY_STRING']}"
-    target = build_admin_url(
-        f"/admin/core/snapshot/replay-auth/?snapshot={snapshot.id}&next={quote(next_path, safe='')}",
-        request=request,
-    )
-    return redirect(target)
-
-
-def _replay_auth_response(request: HttpRequest, snapshot: Snapshot):
-    try:
-        payload = signing.loads(str(request.GET.get("grant") or ""), salt=REPLAY_AUTH_SALT, max_age=REPLAY_GRANT_MAX_AGE)
-    except signing.BadSignature:
-        return _private_snapshot_auth_redirect(request, snapshot, "index.html", preserve_query=False)
-
-    if not isinstance(payload, dict) or not _replay_payload_is_valid(payload, snapshot):
-        return _private_snapshot_auth_redirect(request, snapshot, "index.html", preserve_query=False)
-
-    cookie_value = signing.dumps(payload, salt=REPLAY_AUTH_SALT)
-    response = redirect(_clean_replay_next(request.GET.get("next")))
-    response.set_cookie(
-        _replay_cookie_name(snapshot),
-        cookie_value,
-        max_age=settings.SESSION_COOKIE_AGE,
-        secure=request.is_secure(),
-        httponly=True,
-        samesite="Lax",
-    )
-    return response
-
-
-class SnapshotReplayAuthView(View):
-    """Admin-only handoff that lets a snap host mint its own replay cookie.
-
-    admin.* cannot set a host-only cookie for snap-* (browsers forbid that), and
-    widening the real Django session cookie to *.archivebox.localhost would let XSS
-    in replayed pages hit the admin UI. Instead admin.* proves the user is logged in
-    with a short URL grant, then snap-* validates it and sets a snap-host-only cookie.
-    """
-
-    def get(self, request: HttpRequest):
-        if not is_admin_user(request):
-            return redirect(f"{build_admin_url('/admin/login/', request=request)}?next={quote(request.get_full_path(), safe='')}")
-
-        snapshot = _find_snapshot_by_ref(str(request.GET.get("snapshot") or ""))
-        if not snapshot:
-            raise Http404
-
-        payload = {
-            "snapshot_id": str(snapshot.id),
-            "user_id": str(request.user.pk),
-            "session_key": request.session.session_key,
-            "auth_hash": request.user.get_session_auth_hash(),
-        }
-        grant = signing.dumps(payload, salt=REPLAY_AUTH_SALT)
-        next_path = _clean_replay_next(request.GET.get("next"))
-        target = build_snapshot_url(str(snapshot.id), "_auth", request=request, config=get_request_config(request))
-        return redirect(f"{target}?grant={quote(grant, safe='')}&next={quote(next_path, safe='')}")

archivebox/core/widgets.py

@@ -6,7 +6,6 @@ import hashlib
 from django import forms
 from django.db.models.manager import BaseManager
 from django.db.models.query import QuerySet
-from django.template.loader import render_to_string
 from django.utils.html import escape
 from django.utils.safestring import mark_safe
 
@@ -20,7 +19,11 @@ class TagEditorWidget(forms.Widget):
     - Uses AJAX for autocomplete and tag creation
     """
 
-    template_name = "admin/widgets/tags.html"
+    template_name = ""  # We render manually
+
+    class Media:
+        css = {"all": []}
+        js = []
 
     def __init__(self, attrs=None, snapshot_id=None):
         self.snapshot_id = snapshot_id
@@ -37,6 +40,16 @@ class TagEditorWidget(forms.Widget):
             normalized = f"t_{normalized}"
         return normalized
 
+    def _json_for_inline_script(self, value):
+        """Serialize JSON so it cannot close the surrounding inline <script> tag."""
+        return json.dumps(value).translate(
+            {
+                ord(">"): "\\u003E",
+                ord("<"): "\\u003C",
+                ord("&"): "\\u0026",
+            },
+        )
+
     def _tag_style(self, value):
         """Compute a stable pastel color style for a tag value."""
         tag = (value or "").strip().lower()
@@ -93,10 +106,293 @@ class TagEditorWidget(forms.Widget):
         widget_id_raw = attrs.get("id", name) if attrs else name
         widget_id = self._normalize_id(widget_id_raw)
 
-        return render_to_string(
-            self.template_name,
-            {"name": name, "widget_id": widget_id, "tags_id": f"{widget_id}_tags", "tags": tags},
-        )
+        # Build pills HTML
+        pills_html = ""
+        for tag in tags:
+            pills_html += f'''
+                <span class="tag-pill" data-tag="{self._escape(tag)}" style="{self._tag_style(tag)}">
+                    {self._escape(tag)}
+                    <button type="button" class="tag-remove-btn" data-tag-name="{self._escape(tag)}">&times;</button>
+                </span>
+            '''
+
+        tags_json = self._json_for_inline_script(tags)
+
+        # Build the widget HTML
+        html = f'''
+        <div id="{widget_id}_container" class="tag-editor-container" onclick="focusTagInput_{widget_id}(event)">
+            <div id="{widget_id}_pills" class="tag-pills">
+                {pills_html}
+            </div>
+            <input type="text"
+                   id="{widget_id}_input"
+                   class="tag-inline-input"
+                   list="{widget_id}_datalist"
+                   placeholder="Add tag..."
+                   autocomplete="off"
+                   onkeydown="handleTagKeydown_{widget_id}(event)"
+                   onkeypress="if(event.key==='Enter' || event.keyCode===13 || event.key===' ' || event.code==='Space' || event.key==='Spacebar'){{event.preventDefault(); event.stopPropagation();}}"
+                   oninput="fetchTagAutocomplete_{widget_id}(this.value)"
+            >
+            <datalist id="{widget_id}_datalist"></datalist>
+            <input type="hidden" name="{name}" id="{widget_id}" value="{self._escape(",".join(tags))}">
+        </div>
+
+        <script>
+        (function() {{
+            var currentTags_{widget_id} = {tags_json};
+            var autocompleteTimeout_{widget_id} = null;
+
+            window.focusTagInput_{widget_id} = function(event) {{
+                if (event.target.classList.contains('tag-remove-btn')) return;
+                document.getElementById('{widget_id}_input').focus();
+            }};
+
+            window.updateHiddenInput_{widget_id} = function() {{
+                var hiddenInput = document.getElementById('{widget_id}');
+                if (!hiddenInput) {{
+                    return;
+                }}
+                hiddenInput.value = currentTags_{widget_id}.join(',');
+                hiddenInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
+                hiddenInput.dispatchEvent(new Event('change', {{ bubbles: true }}));
+            }};
+
+            function normalizeTags_{widget_id}(value) {{
+                var rawTags = Array.isArray(value) ? value : String(value || '').split(',');
+                var seen = {{}};
+                return rawTags
+                    .map(function(tag) {{ return String(tag || '').trim(); }})
+                    .filter(function(tag) {{
+                        if (!tag) return false;
+                        var normalized = tag.toLowerCase();
+                        if (seen[normalized]) return false;
+                        seen[normalized] = true;
+                        return true;
+                    }})
+                    .sort(function(a, b) {{
+                        return a.toLowerCase().localeCompare(b.toLowerCase());
+                    }});
+            }}
+
+            window.setTags_{widget_id} = function(value, options) {{
+                currentTags_{widget_id} = normalizeTags_{widget_id}(value);
+                rebuildPills_{widget_id}();
+                if (!(options && options.skipHiddenUpdate)) {{
+                    updateHiddenInput_{widget_id}();
+                }}
+            }};
+
+            window.syncTagEditorFromHidden_{widget_id} = function() {{
+                var hiddenInput = document.getElementById('{widget_id}');
+                if (!hiddenInput) {{
+                    return;
+                }}
+                setTags_{widget_id}(hiddenInput.value, {{ skipHiddenUpdate: true }});
+            }};
+
+            function computeTagStyle_{widget_id}(tagName) {{
+                var hash = 0;
+                var name = String(tagName || '').toLowerCase();
+                for (var i = 0; i < name.length; i++) {{
+                    hash = (hash * 31 + name.charCodeAt(i)) % 360;
+                }}
+                var bg = 'hsl(' + hash + ', 70%, 92%)';
+                var border = 'hsl(' + hash + ', 60%, 82%)';
+                var fg = 'hsl(' + hash + ', 35%, 28%)';
+                return {{ bg: bg, border: border, fg: fg }};
+            }}
+
+            function applyTagStyle_{widget_id}(el, tagName) {{
+                var colors = computeTagStyle_{widget_id}(tagName);
+                el.style.setProperty('--tag-bg', colors.bg);
+                el.style.setProperty('--tag-border', colors.border);
+                el.style.setProperty('--tag-fg', colors.fg);
+            }}
+
+            function getApiKey() {{
+                return (window.ARCHIVEBOX_API_KEY || '').trim();
+            }}
+
+            function buildApiUrl(path) {{
+                var apiKey = getApiKey();
+                if (!apiKey) return path;
+                var sep = path.indexOf('?') !== -1 ? '&' : '?';
+                return path + sep + 'api_key=' + encodeURIComponent(apiKey);
+            }}
+
+            function buildApiHeaders() {{
+                var headers = {{
+                    'Content-Type': 'application/json',
+                }};
+                var apiKey = getApiKey();
+                if (apiKey) headers['X-ArchiveBox-API-Key'] = apiKey;
+                var csrfToken = getCSRFToken();
+                if (csrfToken) headers['X-CSRFToken'] = csrfToken;
+                return headers;
+            }}
+
+            window.addTag_{widget_id} = function(tagName) {{
+                tagName = tagName.trim();
+                if (!tagName) return;
+
+                // Check if tag already exists (case-insensitive)
+                var exists = currentTags_{widget_id}.some(function(t) {{
+                    return t.toLowerCase() === tagName.toLowerCase();
+                }});
+                if (exists) {{
+                    document.getElementById('{widget_id}_input').value = '';
+                    return;
+                }}
+
+                // Add to current tags
+                currentTags_{widget_id}.push(tagName);
+                currentTags_{widget_id} = normalizeTags_{widget_id}(currentTags_{widget_id});
+
+                // Rebuild pills
+                rebuildPills_{widget_id}();
+                updateHiddenInput_{widget_id}();
+
+                // Clear input
+                document.getElementById('{widget_id}_input').value = '';
+
+                // Create tag via API if it doesn't exist (fire and forget)
+                fetch(buildApiUrl('/api/v1/core/tags/create/'), {{
+                    method: 'POST',
+                    headers: buildApiHeaders(),
+                    body: JSON.stringify({{ name: tagName }})
+                }}).catch(function(err) {{
+                    console.log('Tag creation note:', err);
+                }});
+            }};
+
+            window.removeTag_{widget_id} = function(tagName) {{
+                currentTags_{widget_id} = currentTags_{widget_id}.filter(function(t) {{
+                    return t.toLowerCase() !== tagName.toLowerCase();
+                }});
+                rebuildPills_{widget_id}();
+                updateHiddenInput_{widget_id}();
+            }};
+
+            window.rebuildPills_{widget_id} = function() {{
+                var container = document.getElementById('{widget_id}_pills');
+                container.innerHTML = '';
+                currentTags_{widget_id}.forEach(function(tag) {{
+                    var pill = document.createElement('span');
+                    pill.className = 'tag-pill';
+                    pill.setAttribute('data-tag', tag);
+                    applyTagStyle_{widget_id}(pill, tag);
+
+                    var tagText = document.createTextNode(tag);
+                    pill.appendChild(tagText);
+
+                    var removeBtn = document.createElement('button');
+                    removeBtn.type = 'button';
+                    removeBtn.className = 'tag-remove-btn';
+                    removeBtn.setAttribute('data-tag-name', tag);
+                    removeBtn.innerHTML = '&times;';
+                    pill.appendChild(removeBtn);
+
+                    container.appendChild(pill);
+                }});
+            }};
+
+            // Add event delegation for remove buttons
+            document.getElementById('{widget_id}_pills').addEventListener('click', function(event) {{
+                if (event.target.classList.contains('tag-remove-btn')) {{
+                    var tagName = event.target.getAttribute('data-tag-name');
+                    if (tagName) {{
+                        removeTag_{widget_id}(tagName);
+                    }}
+                }}
+            }});
+
+            document.getElementById('{widget_id}').addEventListener('change', function() {{
+                syncTagEditorFromHidden_{widget_id}();
+            }});
+
+            document.getElementById('{widget_id}').addEventListener('archivebox:sync-tags', function() {{
+                syncTagEditorFromHidden_{widget_id}();
+            }});
+
+            window.handleTagKeydown_{widget_id} = function(event) {{
+                var input = event.target;
+                var value = input.value.trim();
+                var isSpace = event.key === ' ' || event.code === 'Space' || event.key === 'Spacebar';
+                var isEnter = event.key === 'Enter' || event.keyCode === 13;
+                var isComma = event.key === ',';
+
+                if (isEnter || isSpace || isComma) {{
+                    event.preventDefault();
+                    event.stopPropagation();
+                    if (value) {{
+                        // Treat commas and whitespace as tag boundaries.
+                        value.split(/[\\s,]+/).forEach(function(tag) {{
+                            addTag_{widget_id}(tag.trim());
+                        }});
+                    }}
+                    return false;
+                }} else if (event.key === 'Backspace' && !value && currentTags_{widget_id}.length > 0) {{
+                    // Remove last tag on backspace when input is empty
+                    var lastTag = currentTags_{widget_id}.pop();
+                    rebuildPills_{widget_id}();
+                    updateHiddenInput_{widget_id}();
+                }}
+            }};
+
+            window.fetchTagAutocomplete_{widget_id} = function(query) {{
+                if (autocompleteTimeout_{widget_id}) {{
+                    clearTimeout(autocompleteTimeout_{widget_id});
+                }}
+
+                autocompleteTimeout_{widget_id} = setTimeout(function() {{
+                    if (!query || query.length < 1) {{
+                        document.getElementById('{widget_id}_datalist').innerHTML = '';
+                        return;
+                    }}
+
+                    fetch(buildApiUrl('/api/v1/core/tags/autocomplete/?q=' + encodeURIComponent(query)))
+                        .then(function(response) {{ return response.json(); }})
+                        .then(function(data) {{
+                            var datalist = document.getElementById('{widget_id}_datalist');
+                            datalist.innerHTML = '';
+                            (data.tags || []).forEach(function(tag) {{
+                                var option = document.createElement('option');
+                                option.value = tag.name;
+                                datalist.appendChild(option);
+                            }});
+                        }})
+                        .catch(function(err) {{
+                            console.log('Autocomplete error:', err);
+                        }});
+                }}, 150);
+            }};
+
+            function escapeHtml(text) {{
+                var div = document.createElement('div');
+                div.textContent = text;
+                return div.innerHTML;
+            }}
+
+            function getCSRFToken() {{
+                var cookies = document.cookie.split(';');
+                for (var i = 0; i < cookies.length; i++) {{
+                    var cookie = cookies[i].trim();
+                    if (cookie.startsWith('csrftoken=')) {{
+                        return cookie.substring('csrftoken='.length);
+                    }}
+                }}
+                // Fallback to hidden input
+                var input = document.querySelector('input[name="csrfmiddlewaretoken"]');
+                return input ? input.value : '';
+            }}
+
+            syncTagEditorFromHidden_{widget_id}();
+        }})();
+        </script>
+        '''
+
+        return mark_safe(html)
 
 
 class URLFiltersWidget(forms.Widget):
@@ -452,58 +748,3 @@ class InlineTagEditorWidget(TagEditorWidget):
         '''
 
         return mark_safe(html)
-
-
-def render_copy_block(text: str, *, multiline: bool = False, copy_text: str | None = None):
-    return render_to_string(
-        "admin/widgets/copy_block.html",
-        {
-            "text": text,
-            "copy_text": text if copy_text is None else copy_text,
-            "element": "pre" if multiline else "code",
-            "multiline": multiline,
-        },
-    )
-
-
-def render_permissions_badge(permissions, *, url: str, object_name: str):
-    from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_META, normalize_permissions
-
-    permissions = normalize_permissions(permissions)
-    icon, label, fg, bg = PERMISSIONS_META[permissions]
-    return render_to_string(
-        "admin/widgets/permissions_badge.html",
-        {
-            "permissions": permissions,
-            "url": url,
-            "object_name": object_name,
-            "icon": icon,
-            "label": label,
-            "fg": fg,
-            "bg": bg,
-            "choices": [(value, *PERMISSIONS_META[value]) for value, _label in PERMISSIONS_CHOICES],
-        },
-    )
-
-
-def render_snapshot_progress(stats: dict, *, successful_plugins=None):
-    """Render the same hook counts and progress bar on snapshot cards and rows."""
-    from archivebox.plugins.discovery import get_plugin_icon
-
-    counts = {name: int(stats.get(name) or 0) for name in ("total", "succeeded", "failed", "skipped", "noresults", "running")}
-    total = counts["total"]
-    completed = sum(counts[name] for name in ("succeeded", "failed", "skipped", "noresults"))
-    plugins = sorted(successful_plugins or ())
-    return render_to_string(
-        "admin/widgets/snapshot_progress.html",
-        {
-            **counts,
-            "completed": completed,
-            "percent": int(completed / total * 100) if total else 0,
-            "succeeded_percent": int(counts["succeeded"] / total * 100) if total else 0,
-            "failed_percent": int((counts["succeeded"] + counts["failed"]) / total * 100) if total else 0,
-            "show_plugins": successful_plugins is not None,
-            "plugins": [(name, mark_safe(icon)) for name in plugins[:8] if str(icon := get_plugin_icon(name)).strip()],
-            "remaining_plugins": max(len(plugins) - 8, 0),
-        },
-    )

archivebox/crawls/admin.py

@@ -1,27 +1,37 @@
 __package__ = "archivebox.crawls"
 
+import json
+from copy import copy
 from typing import ClassVar
+from urllib.parse import urlencode, urlparse
 
+from django import forms
 from django.contrib import admin, messages
-from django.db.models import Count, Q
+from django.core.paginator import Paginator
+from django.db.models import Count, F, Q
 from django.http import HttpRequest, HttpResponseBadRequest, HttpResponseNotAllowed, JsonResponse
 from django.shortcuts import get_object_or_404, redirect
+from django.template.loader import render_to_string
 from django.template.response import TemplateResponse
 from django.urls import path, reverse
 from django.utils import timezone
-from django.utils.html import format_html, format_html_join
+from django.utils.html import escape, format_html, format_html_join
 from django.utils.safestring import mark_safe
 from django_object_actions import action
 
-from archivebox.core.widgets import render_permissions_badge
-from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
-from archivebox.core.models import Snapshot
+from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
+from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.core.permissions import (
+    PERMISSIONS_CHOICES,
     PERMISSIONS_META,
+    PERMISSIONS_PRIVATE,
+    PERMISSIONS_PUBLIC,
+    PERMISSIONS_UNLISTED,
     PERMISSIONS_VALUES,
+    normalize_permissions,
 )
+from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
 from archivebox.crawls.models import Crawl, CrawlSchedule
-from archivebox.crawls.forms import CrawlAdminForm
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.workers.models import RETRY_AT_MAX
@@ -41,6 +51,554 @@ class MaxDepthListFilter(admin.SimpleListFilter):
         return queryset
 
 
+def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50, prefix="snapshots"):
+    """Render a nice inline list view of snapshots with status, title, URL, and progress."""
+
+    query_param = f"{prefix}_q"
+    status_param = f"{prefix}_status"
+    page_param = f"{prefix}_page"
+    query = (request.GET.get(query_param, "") if request is not None else "").strip()
+    status_filter = (request.GET.get(status_param, "") if request is not None else "").strip()
+    valid_statuses = {choice[0] for choice in Snapshot.StatusChoices.choices}
+
+    filtered_qs = snapshots_qs
+    if query:
+        from archivebox.misc.util import filter_queryset_by_uuid_substring
+
+        id_match_pks = list(filter_queryset_by_uuid_substring(Snapshot.objects.all(), query).values_list("pk", flat=True)[:100])
+        filtered_qs = filtered_qs.filter(Q(pk__in=id_match_pks) | Q(url__icontains=query) | Q(title__icontains=query))
+    if status_filter in valid_statuses:
+        filtered_qs = filtered_qs.filter(status=status_filter)
+
+    # Keep ArchiveResult counters as scalar subqueries so the paginated
+    # Snapshot queryset does not become a join+GROUP BY over every result row.
+    snapshots_qs = filtered_qs.order_by("-created_at").annotate(
+        total_results=ArchiveResult.snapshot_count_expr(),
+        succeeded_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
+        failed_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.FAILED),
+        started_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.STARTED),
+        skipped_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SKIPPED),
+        snapshot_permissions=F("permissions"),
+    )
+
+    page_number = request.GET.get(page_param, 1) if request is not None else 1
+    paginator = Paginator(snapshots_qs, page_size)
+    page_obj = paginator.get_page(page_number)
+    snapshots = page_obj.object_list
+    total_count = paginator.count
+
+    def querystring(**updates):
+        if request is None:
+            return "#"
+        params = request.GET.copy()
+        for key, value in updates.items():
+            if value in (None, ""):
+                params.pop(key, None)
+            else:
+                params[key] = str(value)
+        return f"?{params.urlencode()}" if params else "?"
+
+    preserved_inputs = ""
+    if request is not None:
+        managed_params = {query_param, status_param, page_param}
+        preserved_inputs = "".join(
+            f'<input type="hidden" name="{escape(key)}" value="{escape(value)}">'
+            for key, values in request.GET.lists()
+            if key not in managed_params
+            for value in values
+        )
+
+    status_options = "".join(
+        f'<option value="{escape(value)}"{" selected" if status_filter == value else ""}>{escape(label)}</option>'
+        for value, label in Snapshot.StatusChoices.choices
+    )
+
+    controls = f"""
+        <div class="crawl-snapshots-toolbar" style="display: flex; gap: 10px; align-items: center; justify-content: space-between; flex-wrap: wrap; padding: 10px 12px; background: #f8fafc; border-bottom: 1px solid #e2e8f0;">
+            <form method="get" style="display: flex; gap: 8px; align-items: center; flex: 1 1 540px; margin: 0;">
+                {preserved_inputs}
+                <input type="search" name="{query_param}" value="{escape(query)}" placeholder="Filter snapshots by title, URL, or ID"
+                       style="min-width: 260px; flex: 1 1 360px; padding: 7px 10px; border: 1px solid #cbd5e1; border-radius: 6px;">
+                <select name="{status_param}" style="max-width: 170px; padding: 7px 10px; border: 1px solid #cbd5e1; border-radius: 6px;">
+                    <option value="">All statuses</option>
+                    {status_options}
+                </select>
+                <input type="hidden" name="{page_param}" value="1">
+                <button type="submit" class="button" style="padding: 7px 12px;">Filter</button>
+                {f'<a href="{querystring(**{query_param: None, status_param: None, page_param: None})}" style="font-size: 12px; color: #64748b;">Clear</a>' if query or status_filter else ""}
+            </form>
+            <div style="font-size: 12px; color: #64748b; white-space: nowrap;">
+                {page_obj.start_index() if total_count else 0}-{page_obj.end_index() if total_count else 0} of {total_count}
+            </div>
+        </div>
+    """
+
+    if not snapshots:
+        return mark_safe(f"""
+            <div data-crawl-snapshots-list style="border: 1px solid #ddd; border-radius: 6px; overflow: hidden; max-width: 100%;">
+                {controls}
+                <div style="color: #666; font-style: italic; padding: 12px;">No Snapshots found.</div>
+            </div>
+        """)
+
+    # Status colors matching Django admin and progress monitor
+    status_colors = {
+        "queued": ("#6c757d", "#f8f9fa"),  # gray
+        "started": ("#856404", "#fff3cd"),  # amber
+        "paused": ("#1d4ed8", "#dbeafe"),  # blue
+        "sealed": ("#155724", "#d4edda"),  # green
+        "failed": ("#721c24", "#f8d7da"),  # red
+    }
+
+    rows = []
+    for snapshot in snapshots:
+        status = snapshot.status or "queued"
+        color, bg = status_colors.get(status, ("#6c757d", "#f8f9fa"))
+        permissions = snapshot.snapshot_permissions
+        permission_icon = {
+            PERMISSIONS_PUBLIC: "👁",
+            PERMISSIONS_UNLISTED: "🔗",
+            PERMISSIONS_PRIVATE: "🔒",
+        }[permissions]
+        permission_fg, permission_bg = {
+            PERMISSIONS_PUBLIC: ("#047857", "#d1fae5"),
+            PERMISSIONS_UNLISTED: ("#1d4ed8", "#dbeafe"),
+            PERMISSIONS_PRIVATE: ("#991b1b", "#fee2e2"),
+        }[permissions]
+
+        # Calculate progress
+        total = snapshot.total_results
+        succeeded = snapshot.succeeded_results
+        failed = snapshot.failed_results
+        running = snapshot.started_results
+        skipped = snapshot.skipped_results
+        done = succeeded + failed + skipped
+        pending = max(total - done - running, 0)
+        progress_pct = int((done / total) * 100) if total > 0 else 0
+        progress_text = f"{done}/{total}" if total > 0 else "-"
+        progress_title = f"{succeeded} succeeded, {failed} failed, {running} running, {pending} pending, {skipped} skipped"
+        progress_color = "#28a745"
+        if failed:
+            progress_color = "#dc3545"
+        elif running:
+            progress_color = "#17a2b8"
+        elif pending:
+            progress_color = "#ffc107"
+
+        # Truncate title and URL
+        snapshot_title = snapshot.title or "Untitled"
+        title = snapshot_title[:60]
+        if len(snapshot_title) > 60:
+            title += "..."
+        url_display = snapshot.url[:50]
+        if len(snapshot.url) > 50:
+            url_display += "..."
+        delete_button = ""
+        exclude_button = ""
+        if crawl is not None:
+            delete_url = reverse("admin:crawls_crawl_snapshot_delete", args=[crawl.pk, snapshot.pk])
+            exclude_url = reverse("admin:crawls_crawl_snapshot_exclude_domain", args=[crawl.pk, snapshot.pk])
+            delete_button = f'''
+                <button type="button"
+                        class="crawl-snapshots-action"
+                        data-post-url="{escape(delete_url)}"
+                        data-confirm="Delete this snapshot from the crawl?"
+                        title="Delete this snapshot from the crawl and remove its URL from the crawl queue."
+                        aria-label="Delete snapshot"
+                        style="border: 1px solid #ddd; background: #fff; color: #666; border-radius: 4px; width: 28px; height: 28px; cursor: pointer;">🗑</button>
+            '''
+            exclude_button = f'''
+                <button type="button"
+                        class="crawl-snapshots-action"
+                        data-post-url="{escape(exclude_url)}"
+                        data-confirm="Exclude this domain from the crawl? This removes matching queued URLs, deletes pending matching snapshots, and blocks future matches."
+                        title="Exclude this domain from this crawl. This removes matching URLs from the crawl queue, deletes pending matching snapshots, and blocks future matches."
+                        aria-label="Exclude domain from crawl"
+                        style="border: 1px solid #ddd; background: #fff; color: #666; border-radius: 4px; width: 28px; height: 28px; cursor: pointer;">⊘</button>
+            '''
+
+        # Format date
+        date_str = snapshot.created_at.strftime("%Y-%m-%d %H:%M") if snapshot.created_at else "-"
+
+        rows.append(f'''
+            <tr style="border-bottom: 1px solid #eee;">
+                <td style="padding: 6px 8px; white-space: nowrap;">
+                    <span style="display: inline-block; padding: 2px 8px; border-radius: 10px;
+                                 font-size: 11px; font-weight: 500; text-transform: uppercase;
+                                 color: {color}; background: {bg};">{status}</span>
+                </td>
+                <td style="padding: 6px 8px; white-space: nowrap; text-align: center;">
+                    <span title="{permissions}" style="display:inline-flex; align-items:center; justify-content:center; width:22px; height:22px; border-radius:999px; font-size:12px; color:{permission_fg}; background:{permission_bg};">{permission_icon}</span>
+                </td>
+                <td style="padding: 6px 8px; white-space: nowrap;">
+                    <a href="/{snapshot.archive_path}/" style="text-decoration: none;">
+                        <img src="/{snapshot.archive_path}/favicon.ico"
+                             style="width: 16px; height: 16px; vertical-align: middle; margin-right: 4px;"
+                             onerror="this.style.display='none'"/>
+                    </a>
+                </td>
+                <td style="padding: 6px 8px; max-width: 300px;">
+                    <a href="{snapshot.admin_change_url}" style="color: #417690; text-decoration: none; font-weight: 500;"
+                       title="{escape(snapshot_title)}">{escape(title)}</a>
+                </td>
+                <td style="padding: 6px 8px; max-width: 250px;">
+                    <a href="{escape(snapshot.url)}" target="_blank"
+                       style="color: #666; text-decoration: none; font-family: monospace; font-size: 11px;"
+                       title="{escape(snapshot.url)}">{escape(url_display)}</a>
+                </td>
+                <td style="padding: 6px 8px; white-space: nowrap; text-align: center;">
+                    <div style="display: inline-flex; align-items: center; gap: 6px;" title="{escape(progress_title)}">
+                        <div style="width: 60px; height: 6px; background: #eee; border-radius: 3px; overflow: hidden;">
+                            <div style="width: {progress_pct}%; height: 100%;
+                                        background: {progress_color};
+                                        transition: width 0.3s;"></div>
+                        </div>
+                        <a href="/admin/core/archiveresult/?snapshot__id__exact={snapshot.id}"
+                           style="font-size: 11px; color: #417690; min-width: 35px; text-decoration: none;"
+                           title="View archive results">{progress_text}</a>
+                    </div>
+                </td>
+                <td style="padding: 6px 8px; white-space: nowrap; color: #888; font-size: 11px;">
+                    {date_str}
+                </td>
+                {f'<td style="padding: 6px 8px; white-space: nowrap; text-align: right;"><div style="display: inline-flex; gap: 6px;">{exclude_button}{delete_button}</div></td>' if crawl is not None else ""}
+            </tr>
+        ''')
+
+    pagination = ""
+    if paginator.num_pages > 1:
+        pagination = f"""
+            <div style="display: flex; gap: 10px; align-items: center; justify-content: center; padding: 10px 12px; background: #f8fafc; border-top: 1px solid #e2e8f0; font-size: 12px;">
+                {"<a class='button' style='padding: 5px 10px;' href='" + querystring(**{page_param: page_obj.previous_page_number()}) + "'>Previous</a>" if page_obj.has_previous() else "<span style='color:#94a3b8;'>Previous</span>"}
+                <span style="color: #64748b;">Page {page_obj.number} of {paginator.num_pages}</span>
+                {"<a class='button' style='padding: 5px 10px;' href='" + querystring(**{page_param: page_obj.next_page_number()}) + "'>Next</a>" if page_obj.has_next() else "<span style='color:#94a3b8;'>Next</span>"}
+            </div>
+        """
+
+    return mark_safe(f"""
+        <div data-crawl-snapshots-list style="border: 1px solid #ddd; border-radius: 6px; overflow: hidden; max-width: 100%;">
+            {controls}
+            <table style="width: 100%; border-collapse: collapse; font-size: 13px;">
+                <thead>
+                    <tr style="background: #f5f5f5; border-bottom: 2px solid #ddd;">
+                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">Status</th>
+                        <th style="padding: 8px 4px; text-align: center; font-weight: 600; color: #333; width: 22px;">🔒</th>
+                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333; width: 24px;"></th>
+                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">Title</th>
+                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">URL</th>
+                        <th style="padding: 8px; text-align: center; font-weight: 600; color: #333;">Progress</th>
+                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">Created</th>
+                        {
+        '<th style="padding: 8px; text-align: right; font-weight: 600; color: #333;">Actions</th>' if crawl is not None else ""
+    }
+                    </tr>
+                </thead>
+                <tbody>
+                    {"".join(rows)}
+                </tbody>
+            </table>
+            {pagination}
+        </div>
+        {
+        '''
+        <script>
+        (function() {
+            if (window.__archiveboxCrawlSnapshotActionsBound) {
+                return;
+            }
+            window.__archiveboxCrawlSnapshotActionsBound = true;
+
+            function getCookie(name) {
+                var cookieValue = null;
+                if (!document.cookie) {
+                    return cookieValue;
+                }
+                var cookies = document.cookie.split(';');
+                for (var i = 0; i < cookies.length; i++) {
+                    var cookie = cookies[i].trim();
+                    if (cookie.substring(0, name.length + 1) === (name + '=')) {
+                        cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
+                        break;
+                    }
+                }
+                return cookieValue;
+            }
+
+            document.addEventListener('click', function(event) {
+                var button = event.target.closest('.crawl-snapshots-action');
+                if (!button) {
+                    return;
+                }
+                event.preventDefault();
+
+                var confirmMessage = button.getAttribute('data-confirm');
+                if (confirmMessage && !window.confirm(confirmMessage)) {
+                    return;
+                }
+
+                button.disabled = true;
+
+                fetch(button.getAttribute('data-post-url'), {
+                    method: 'POST',
+                    credentials: 'same-origin',
+                    headers: {
+                        'X-CSRFToken': getCookie('csrftoken') || '',
+                        'X-Requested-With': 'XMLHttpRequest'
+                    }
+                }).then(function(response) {
+                    return response.json().then(function(data) {
+                        if (!response.ok) {
+                            throw new Error(data.error || 'Request failed');
+                        }
+                        return data;
+                    });
+                }).then(function() {
+                    window.location.reload();
+                }).catch(function(error) {
+                    button.disabled = false;
+                    window.alert(error.message || 'Request failed');
+                });
+            });
+        })();
+        </script>
+        '''
+        if crawl is not None
+        else ""
+    }
+    """)
+
+
+class URLFiltersField(forms.Field):
+    widget = URLFiltersWidget(source_selector="#id_urls")
+
+    def to_python(self, value):
+        if isinstance(value, dict):
+            return value
+        return {"allowlist": "", "denylist": "", "same_domain_only": False, "subpaths_only": False, "only_new": False}
+
+
+class CrawlAdminForm(forms.ModelForm):
+    """Custom form for Crawl admin to render urls field as textarea."""
+
+    tags_editor = forms.CharField(
+        label="Tags",
+        required=False,
+        widget=TagEditorWidget(),
+        help_text="Type tag names and press Enter or Space to add. Click × to remove.",
+    )
+    url_filters = URLFiltersField(
+        label="URL Filters",
+        required=False,
+        help_text="Set URL_ALLOWLIST / URL_DENYLIST for this crawl.",
+    )
+
+    class Meta:
+        model = Crawl
+        fields = "__all__"
+        widgets: ClassVar[dict[str, forms.Widget]] = {
+            "urls": forms.Textarea(
+                attrs={
+                    "rows": 8,
+                    "style": "width: 100%; font-family: monospace; font-size: 13px;",
+                    "placeholder": "https://example.com\nhttps://example2.com\n# Comments start with #",
+                },
+            ),
+            "notes": forms.Textarea(
+                attrs={
+                    "rows": 1,
+                    "style": "width: 100%; min-height: 0; resize: vertical;",
+                },
+            ),
+        }
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        config = dict(self.instance.config or {}) if self.instance and self.instance.pk else {}
+        if self.instance and self.instance.pk:
+            self.initial["tags_editor"] = self.instance.tags_str
+        effective_only_new = self.effective_only_new(self.instance if self.instance and self.instance.pk else None)
+        derived_filter_toggles = self.derive_filter_toggles(
+            self.instance.urls if self.instance and self.instance.pk else "",
+            config.get("URL_ALLOWLIST", ""),
+        )
+        self.initial["url_filters"] = {
+            "allowlist": config.get("URL_ALLOWLIST", ""),
+            "denylist": config.get("URL_DENYLIST", ""),
+            "same_domain_only": derived_filter_toggles["same_domain_only"],
+            "subpaths_only": derived_filter_toggles["subpaths_only"],
+            "only_new": effective_only_new,
+        }
+
+    @staticmethod
+    def extract_url_line(line):
+        line = str(line or "").strip()
+        if not line or line.startswith("#"):
+            return ""
+        if line.startswith("{"):
+            try:
+                return str(json.loads(line).get("url", "")).strip()
+            except (TypeError, ValueError, json.JSONDecodeError):
+                return ""
+        return line
+
+    @staticmethod
+    def regex_escape(text):
+        escaped = ""
+        for char in str(text or ""):
+            escaped += f"\\{char}" if char in r".*+?^${}()|[]\\" else char
+        return escaped
+
+    @classmethod
+    def generated_host_allowlist(cls, urls):
+        seen = set()
+        domains = []
+        for raw_line in str(urls or "").splitlines():
+            url = cls.extract_url_line(raw_line)
+            if not url:
+                continue
+            parsed = urlparse(url)
+            domain = (parsed.hostname or "").lower()
+            if not domain or domain in seen:
+                continue
+            seen.add(domain)
+            domains.append(domain)
+        if not domains:
+            return ""
+        return "^https?://(" + "|".join(cls.regex_escape(domain) for domain in domains) + ")([:/]|$)"
+
+    @staticmethod
+    def subpath_prefix(pathname):
+        path = str(pathname or "/")
+        while "//" in path:
+            path = path.replace("//", "/")
+        if not path or path == "/":
+            return "/"
+        if path.endswith("/"):
+            return path
+        last_slash = path.rfind("/")
+        last_part = path[last_slash + 1 :]
+        if "." in last_part:
+            return path[: last_slash + 1] or "/"
+        return path
+
+    @staticmethod
+    def parsed_host_and_port(parsed):
+        host = (parsed.hostname or "").lower()
+        if not host:
+            return ""
+        try:
+            port = parsed.port
+        except ValueError:
+            port = None
+        return f"{host}:{port}" if port is not None else host
+
+    @classmethod
+    def generated_subpath_allowlist(cls, urls):
+        seen = set()
+        paths = []
+        for raw_line in str(urls or "").splitlines():
+            url = cls.extract_url_line(raw_line)
+            if not url:
+                continue
+            parsed = urlparse(url)
+            domain = (parsed.hostname or "").lower()
+            if domain:
+                seen.add(domain)
+            host = cls.parsed_host_and_port(parsed)
+            path = cls.subpath_prefix(parsed.path)
+            path_key = f"{host}{path}"
+            if not host or path_key in seen:
+                continue
+            seen.add(path_key)
+            paths.append((host, path))
+        if not paths:
+            return ""
+        patterns = []
+        for host, path in paths:
+            if path == "/":
+                patterns.append(f"^https?://{cls.regex_escape(host)}([/?#]|$)")
+            elif path.endswith("/"):
+                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}")
+            else:
+                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}([/?#]|$)")
+        return "\n".join(patterns)
+
+    @classmethod
+    def derive_filter_toggles(cls, urls, allowlist):
+        normalized_allowlist = "\n".join(Crawl.split_filter_patterns(allowlist))
+        if not normalized_allowlist:
+            return {"same_domain_only": False, "subpaths_only": False}
+        if normalized_allowlist == cls.generated_subpath_allowlist(urls):
+            return {"same_domain_only": True, "subpaths_only": True}
+        if normalized_allowlist == cls.generated_host_allowlist(urls):
+            return {"same_domain_only": True, "subpaths_only": False}
+        return {"same_domain_only": False, "subpaths_only": False}
+
+    @staticmethod
+    def effective_only_new(crawl=None):
+        from archivebox.config.common import get_config
+
+        if crawl is not None:
+            return bool(get_config(crawl=crawl, resolve_plugins=False).ONLY_NEW)
+        return bool(get_config(resolve_plugins=False).ONLY_NEW)
+
+    @staticmethod
+    def inherited_only_new(crawl):
+        crawl_without_only_new = copy(crawl)
+        config = dict(crawl.config or {})
+        config.pop("ONLY_NEW", None)
+        crawl_without_only_new.config = config
+        return CrawlAdminForm.effective_only_new(crawl_without_only_new)
+
+    def clean_tags_editor(self):
+        tags_str = self.cleaned_data.get("tags_editor", "")
+        tag_names = []
+        seen = set()
+        for raw_name in tags_str.split(","):
+            name = raw_name.strip()
+            if not name:
+                continue
+            lowered = name.lower()
+            if lowered in seen:
+                continue
+            seen.add(lowered)
+            tag_names.append(name)
+        return ",".join(tag_names)
+
+    def clean_url_filters(self):
+        value = self.cleaned_data.get("url_filters") or {}
+        return {
+            "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
+            "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
+            "same_domain_only": bool(value.get("same_domain_only")),
+            "subpaths_only": bool(value.get("subpaths_only")),
+            "only_new": bool(value.get("only_new")),
+        }
+
+    def save(self, commit=True):
+        instance = super().save(commit=False)
+        instance.tags_str = self.cleaned_data.get("tags_editor", "")
+        if f"{self.add_prefix('url_filters')}_allowlist" in self.data or f"{self.add_prefix('url_filters')}_denylist" in self.data:
+            url_filters = self.cleaned_data.get("url_filters") or {}
+            instance.set_url_filters(
+                url_filters.get("allowlist", ""),
+                url_filters.get("denylist", ""),
+            )
+            config = dict(instance.config or {})
+            only_new = bool(url_filters.get("only_new"))
+            inherited_only_new = self.inherited_only_new(instance)
+            if only_new != inherited_only_new:
+                config["ONLY_NEW"] = only_new
+            else:
+                config.pop("ONLY_NEW", None)
+            instance.config = config
+        if commit:
+            instance.save()
+            instance.apply_crawl_config_filters()
+            self._save_m2m()
+        return instance
+
+
 class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
     form = CrawlAdminForm
     change_form_template = "admin/crawls/crawl/change_form.html"
@@ -88,7 +646,13 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
     readonly_fields = ("created_at", "modified_at", "stop_reason_display")
 
     fieldsets = (
-        card_fieldset("URLs", ("urls", "url_filters"), wide=True),
+        (
+            "URLs",
+            {
+                "fields": ("urls", "url_filters"),
+                "classes": ("card", "wide"),
+            },
+        ),
         (
             "Overview",
             {
@@ -110,7 +674,13 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         ),
     )
     add_fieldsets = (
-        card_fieldset("URLs", ("urls", "url_filters"), wide=True),
+        (
+            "URLs",
+            {
+                "fields": ("urls", "url_filters"),
+                "classes": ("card", "wide"),
+            },
+        ),
         (
             "Overview",
             {
@@ -454,11 +1024,45 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="👁", ordering="permissions")
     def permissions_badge(self, obj):
-        permissions = obj.permissions
-        return render_permissions_badge(
+        permissions = normalize_permissions(obj.permissions)
+        icon, label, fg, bg = PERMISSIONS_META[permissions]
+        menu_items = format_html_join(
+            "",
+            (
+                '<button type="button" class="snapshot-permissions-menu-item{}" data-permissions="{}">'
+                '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
+                "<span>{}</span>"
+                "</button>"
+            ),
+            (
+                (
+                    " is-active" if choice_value == permissions else "",
+                    choice_value,
+                    choice_fg,
+                    choice_bg,
+                    choice_icon,
+                    choice_label,
+                )
+                for choice_value, choice_label in PERMISSIONS_CHOICES
+                for choice_icon, _choice_title, choice_fg, choice_bg in [PERMISSIONS_META[choice_value]]
+            ),
+        )
+        return format_html(
+            '<span class="snapshot-permissions-quick" data-current-permissions="{}" data-permissions-url="{}">'
+            '<button type="button" class="snapshot-permissions-button snapshot-permissions-{}" title="{}" aria-label="Change crawl permissions: {}" aria-expanded="false">'
+            '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
+            "</button>"
+            '<span class="snapshot-permissions-menu" role="menu" hidden>{}</span>'
+            "</span>",
             permissions,
-            url=reverse(f"{self.admin_site.name}:crawls_crawl_set_permissions", args=[obj.pk]),
-            object_name="crawl",
+            reverse(f"{self.admin_site.name}:crawls_crawl_set_permissions", args=[obj.pk]),
+            permissions,
+            label,
+            label,
+            fg,
+            bg,
+            icon,
+            menu_items,
         )
 
     @admin.display(description="Pause")
@@ -485,11 +1089,36 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Snapshots")
     def snapshots_changelist(self, obj):
-        return self.admin_site._registry[Snapshot].render_embedded_changelist(
-            self.request,
-            filters={"crawl_id": str(obj.pk)},
-            title="Snapshots in this crawl",
+        request = self.request
+        snapshot_changelist = reverse("admin:core_snapshot_changelist")
+        scoped_params = {"crawl_id": str(obj.pk)}
+        full_url = f"{snapshot_changelist}?{urlencode(scoped_params)}"
+
+        snapshot_admin = self.admin_site._registry[Snapshot]
+        changelist_request = copy(request)
+        changelist_request.method = "GET"
+        changelist_request.path = snapshot_changelist
+        changelist_request.GET = request.GET.copy()
+        changelist_request.GET.update(
+            {
+                **scoped_params,
+                "_embedded": "crawl",
+                "per_page": "200",
+            },
         )
+        changelist_request.POST = request.POST.copy()
+        changelist_request.POST.clear()
+
+        response = snapshot_admin.changelist_view(
+            changelist_request,
+            extra_context={"embedded_changelist": True},
+        )
+        context = {
+            **response.context_data,
+            "snapshot_changelist_url": full_url,
+            "crawl": obj,
+        }
+        return mark_safe(render_to_string("admin/crawls/crawl/snapshots_changelist.html", context, request=request))
 
     def delete_snapshot_view(self, request: HttpRequest, object_id: str, snapshot_id: str):
         if request.method != "POST":
@@ -549,25 +1178,83 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         first_url = next((line.strip() for line in (obj.urls or "").splitlines() if line.strip() and not line.strip().startswith("#")), "")
         return first_url[:80] + "..." if len(first_url) > 80 else first_url
 
+    @admin.display(description="URLs")
+    def urls_editor(self, obj):
+        """Editor for crawl URLs."""
+        widget_id = f"crawl_urls_{obj.pk}"
+
+        # Escape for safe HTML embedding
+        escaped_urls = (obj.urls or "").replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
+
+        # Count lines for auto-expand logic
+        line_count = len((obj.urls or "").split("\n"))
+        uri_rows = min(max(3, line_count), 10)
+
+        html = f'''
+        <div id="{widget_id}_container" style="max-width: 900px;">
+            <!-- URLs input -->
+            <div style="margin-bottom: 12px;">
+                <label style="font-weight: bold; display: block; margin-bottom: 4px;">URLs (one per line):</label>
+                <textarea id="{widget_id}_urls"
+                          style="width: 100%; font-family: monospace; font-size: 13px;
+                                 padding: 8px; border: 1px solid #ccc; border-radius: 4px;
+                                 resize: vertical;"
+                          rows="{uri_rows}"
+                          placeholder="https://example.com&#10;https://example2.com&#10;# Comments start with #"
+                          readonly>{escaped_urls}</textarea>
+                <p style="color: #666; font-size: 12px; margin: 4px 0 0 0;">
+                    {line_count} URL{"s" if line_count != 1 else ""} · Note: URLs displayed here for reference only
+                </p>
+            </div>
+        </div>
+        '''
+        return mark_safe(html)
+
 
 class CrawlScheduleAdmin(BaseModelAdmin):
-    change_form_template = "admin/crawls/crawlschedule/change_form.html"
-
-    class Media:
-        css: ClassVar[dict[str, tuple[str, ...]]] = {"all": ("admin/crawls/crawl_change.css",)}
-
     list_display = ("id", "created_at", "created_by", "label", "notes", "template_str", "crawls", "num_crawls", "num_snapshots")
     sort_fields = ("id", "created_at", "created_by", "label", "notes", "template_str")
     search_fields = ("id", "created_by__username", "label", "notes", "schedule_id", "template_id", "template__urls")
 
-    readonly_fields = ("created_at", "modified_at", "crawls")
+    readonly_fields = ("created_at", "modified_at", "crawls", "snapshots")
     autocomplete_fields = ("template", "created_by")
 
     fieldsets = (
-        card_fieldset("Schedule Info", ("label", "notes")),
-        card_fieldset("Configuration", ("schedule", "template")),
-        card_fieldset("Metadata", ("created_by", "created_at", "modified_at")),
-        card_fieldset("Crawls", ("crawls",), wide=True),
+        (
+            "Schedule Info",
+            {
+                "fields": ("label", "notes"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Configuration",
+            {
+                "fields": ("is_enabled", "schedule", "template"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Metadata",
+            {
+                "fields": ("created_by", "created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Crawls",
+            {
+                "fields": ("crawls",),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Snapshots",
+            {
+                "fields": ("snapshots",),
+                "classes": ("card", "wide"),
+            },
+        ),
     )
 
     list_filter = ("created_by",)
@@ -588,23 +1275,29 @@ class CrawlScheduleAdmin(BaseModelAdmin):
         )
 
     def change_view(self, request, object_id, form_url="", extra_context=None):
-        schedule = self.get_object(request, object_id)
-        extra_context = {**(extra_context or {})}
-        if schedule:
-            extra_context["schedule_snapshots_changelist"] = self.admin_site._registry[Snapshot].render_embedded_changelist(
-                request,
-                filters={"crawl__schedule__id__exact": str(schedule.pk)},
-                title="Snapshots in this schedule",
-                default_search_mode="meta",
-            )
-        return super().change_view(request, object_id, form_url, extra_context)
+        self.request = request
+        from django.db import connections, router
+
+        connection = connections[router.db_for_write(self.model)]
+        if request.method != "POST" or connection.vendor != "sqlite":
+            return super().change_view(request, object_id, form_url, extra_context)
+        # Reserve the writer before Django's atomic admin form reads. A
+        # deferred read transaction cannot safely upgrade while the runner
+        # writes, even when SQLite's existing busy timeout has time left.
+        connection.ensure_connection()
+        previous_mode = connection.transaction_mode
+        connection.transaction_mode = "IMMEDIATE"
+        try:
+            return super().change_view(request, object_id, form_url, extra_context)
+        finally:
+            connection.transaction_mode = previous_mode
 
     def add_view(self, request, form_url="", extra_context=None):
         return redirect("/add/#schedule")
 
     def get_fieldsets(self, request, obj=None):
         if obj is None:
-            return tuple(fieldset for fieldset in self.fieldsets if fieldset[0] != "Crawls")
+            return tuple(fieldset for fieldset in self.fieldsets if fieldset[0] not in {"Crawls", "Snapshots"})
         return self.fieldsets
 
     def save_model(self, request, obj, form, change):
@@ -637,6 +1330,14 @@ class CrawlScheduleAdmin(BaseModelAdmin):
             ((crawl.admin_change_url, crawl) for crawl in obj.crawl_set.all().order_by("-created_at")[:20]),
         ) or mark_safe("<i>No Crawls yet...</i>")
 
+    def snapshots(self, obj):
+        crawl_ids = obj.crawl_set.values_list("pk", flat=True)
+        return render_snapshots_list(
+            Snapshot.objects.filter(crawl_id__in=crawl_ids),
+            request=self.request,
+            prefix="schedule_snapshots",
+        )
+
 
 def register_admin(admin_site):
     admin_site.register(Crawl, CrawlAdmin)

archivebox/crawls/forms.py

@@ -1,237 +0,0 @@
-"""Crawl form input parsing; lifecycle operations remain on Crawl."""
-
-import json
-from copy import copy
-from typing import ClassVar
-from urllib.parse import urlparse
-
-from django import forms
-
-from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
-from archivebox.crawls.models import Crawl
-
-
-class URLFiltersField(forms.Field):
-    widget = URLFiltersWidget(source_selector="#id_urls")
-
-    def to_python(self, value):
-        value = value if isinstance(value, dict) else {}
-        return {
-            "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
-            "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
-            "same_domain_only": bool(value.get("same_domain_only")),
-            "subpaths_only": bool(value.get("subpaths_only")),
-            "only_new": bool(value.get("only_new")),
-        }
-
-
-class CrawlAdminForm(forms.ModelForm):
-    """Custom form for Crawl admin to render urls field as textarea."""
-
-    tags_editor = forms.CharField(
-        label="Tags",
-        required=False,
-        widget=TagEditorWidget(),
-        help_text="Type tag names and press Enter or Space to add. Click × to remove.",
-    )
-    url_filters = URLFiltersField(
-        label="URL Filters",
-        required=False,
-        help_text="Set URL_ALLOWLIST / URL_DENYLIST for this crawl.",
-    )
-
-    class Meta:
-        model = Crawl
-        fields = "__all__"
-        widgets: ClassVar[dict[str, forms.Widget]] = {
-            "urls": forms.Textarea(
-                attrs={
-                    "rows": 8,
-                    "style": "width: 100%; font-family: monospace; font-size: 13px;",
-                    "placeholder": "https://example.com\nhttps://example2.com\n# Comments start with #",
-                },
-            ),
-            "notes": forms.Textarea(
-                attrs={
-                    "rows": 1,
-                    "style": "width: 100%; min-height: 0; resize: vertical;",
-                },
-            ),
-        }
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        config = dict(self.instance.config or {}) if self.instance and self.instance.pk else {}
-        if self.instance and self.instance.pk:
-            self.initial["tags_editor"] = self.instance.tags_str
-        effective_only_new = self.effective_only_new(self.instance if self.instance and self.instance.pk else None)
-        derived_filter_toggles = self.derive_filter_toggles(
-            self.instance.urls if self.instance and self.instance.pk else "",
-            config.get("URL_ALLOWLIST", ""),
-        )
-        self.initial["url_filters"] = {
-            "allowlist": config.get("URL_ALLOWLIST", ""),
-            "denylist": config.get("URL_DENYLIST", ""),
-            "same_domain_only": derived_filter_toggles["same_domain_only"],
-            "subpaths_only": derived_filter_toggles["subpaths_only"],
-            "only_new": effective_only_new,
-        }
-
-    @staticmethod
-    def extract_url_line(line):
-        line = str(line or "").strip()
-        if not line or line.startswith("#"):
-            return ""
-        if line.startswith("{"):
-            try:
-                return str(json.loads(line).get("url", "")).strip()
-            except (TypeError, ValueError, json.JSONDecodeError):
-                return ""
-        return line
-
-    @staticmethod
-    def regex_escape(text):
-        escaped = ""
-        for char in str(text or ""):
-            escaped += f"\\{char}" if char in r".*+?^${}()|[]\\" else char
-        return escaped
-
-    @classmethod
-    def generated_host_allowlist(cls, urls):
-        seen = set()
-        domains = []
-        for raw_line in str(urls or "").splitlines():
-            url = cls.extract_url_line(raw_line)
-            if not url:
-                continue
-            parsed = urlparse(url)
-            domain = (parsed.hostname or "").lower()
-            if not domain or domain in seen:
-                continue
-            seen.add(domain)
-            domains.append(domain)
-        if not domains:
-            return ""
-        return "^https?://(" + "|".join(cls.regex_escape(domain) for domain in domains) + ")([:/]|$)"
-
-    @staticmethod
-    def subpath_prefix(pathname):
-        path = str(pathname or "/")
-        while "//" in path:
-            path = path.replace("//", "/")
-        if not path or path == "/":
-            return "/"
-        if path.endswith("/"):
-            return path
-        last_slash = path.rfind("/")
-        last_part = path[last_slash + 1 :]
-        if "." in last_part:
-            return path[: last_slash + 1] or "/"
-        return path
-
-    @staticmethod
-    def parsed_host_and_port(parsed):
-        host = (parsed.hostname or "").lower()
-        if not host:
-            return ""
-        try:
-            port = parsed.port
-        except ValueError:
-            port = None
-        return f"{host}:{port}" if port is not None else host
-
-    @classmethod
-    def generated_subpath_allowlist(cls, urls):
-        seen = set()
-        paths = []
-        for raw_line in str(urls or "").splitlines():
-            url = cls.extract_url_line(raw_line)
-            if not url:
-                continue
-            parsed = urlparse(url)
-            domain = (parsed.hostname or "").lower()
-            if domain:
-                seen.add(domain)
-            host = cls.parsed_host_and_port(parsed)
-            path = cls.subpath_prefix(parsed.path)
-            path_key = f"{host}{path}"
-            if not host or path_key in seen:
-                continue
-            seen.add(path_key)
-            paths.append((host, path))
-        if not paths:
-            return ""
-        patterns = []
-        for host, path in paths:
-            if path == "/":
-                patterns.append(f"^https?://{cls.regex_escape(host)}([/?#]|$)")
-            elif path.endswith("/"):
-                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}")
-            else:
-                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}([/?#]|$)")
-        return "\n".join(patterns)
-
-    @classmethod
-    def derive_filter_toggles(cls, urls, allowlist):
-        normalized_allowlist = "\n".join(Crawl.split_filter_patterns(allowlist))
-        if not normalized_allowlist:
-            return {"same_domain_only": False, "subpaths_only": False}
-        if normalized_allowlist == cls.generated_subpath_allowlist(urls):
-            return {"same_domain_only": True, "subpaths_only": True}
-        if normalized_allowlist == cls.generated_host_allowlist(urls):
-            return {"same_domain_only": True, "subpaths_only": False}
-        return {"same_domain_only": False, "subpaths_only": False}
-
-    @staticmethod
-    def effective_only_new(crawl=None):
-        from archivebox.config.common import get_config
-
-        if crawl is not None:
-            return bool(get_config(crawl=crawl, resolve_plugins=False).ONLY_NEW)
-        return bool(get_config(resolve_plugins=False).ONLY_NEW)
-
-    @staticmethod
-    def inherited_only_new(crawl):
-        crawl_without_only_new = copy(crawl)
-        config = dict(crawl.config or {})
-        config.pop("ONLY_NEW", None)
-        crawl_without_only_new.config = config
-        return CrawlAdminForm.effective_only_new(crawl_without_only_new)
-
-    def clean_tags_editor(self):
-        tags_str = self.cleaned_data.get("tags_editor", "")
-        tag_names = []
-        seen = set()
-        for raw_name in tags_str.split(","):
-            name = raw_name.strip()
-            if not name:
-                continue
-            lowered = name.lower()
-            if lowered in seen:
-                continue
-            seen.add(lowered)
-            tag_names.append(name)
-        return ",".join(tag_names)
-
-    def save(self, commit=True):
-        instance = super().save(commit=False)
-        instance.tags_str = self.cleaned_data.get("tags_editor", "")
-        if f"{self.add_prefix('url_filters')}_allowlist" in self.data or f"{self.add_prefix('url_filters')}_denylist" in self.data:
-            url_filters = self.cleaned_data.get("url_filters") or {}
-            instance.set_url_filters(
-                url_filters.get("allowlist", ""),
-                url_filters.get("denylist", ""),
-            )
-            config = dict(instance.config or {})
-            only_new = bool(url_filters.get("only_new"))
-            inherited_only_new = self.inherited_only_new(instance)
-            if only_new != inherited_only_new:
-                config["ONLY_NEW"] = only_new
-            else:
-                config.pop("ONLY_NEW", None)
-            instance.config = config
-        if commit:
-            instance.save()
-            instance.apply_crawl_config_filters()
-            self._save_m2m()
-        return instance

archivebox/crawls/models.py

@@ -1,6 +1,6 @@
 __package__ = "archivebox.crawls"
 
-from typing import TYPE_CHECKING, Any, ClassVar
+from typing import TYPE_CHECKING, Any
 from collections.abc import Iterable, Mapping
 import uuid
 import json
@@ -31,7 +31,7 @@ from archivebox.base_models.models import (
     ModelWithHealthStats,
     get_or_create_system_user_pk,
 )
-from archivebox.workers.models import DefaultStatusChoices, ModelWithQueue
+from archivebox.workers.models import ModelWithQueue
 from archivebox.crawls.schedule_util import next_run_for_schedule, validate_schedule
 from archivebox.misc.util import parse_date, sanitize_html_text, validate_url, validate_url_length
 
@@ -184,7 +184,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     retry_at_field_name = "retry_at"
     state_field_name = "status"
-    StatusChoices: ClassVar[type[DefaultStatusChoices]] = DefaultStatusChoices
+    StatusChoices = ModelWithQueue.StatusChoices
     INITIAL_STATE = StatusChoices.QUEUED
     ACTIVE_STATE = StatusChoices.STARTED
     FINAL_STATES = (StatusChoices.SEALED,)

archivebox/machine/admin.py

@@ -1,5 +1,6 @@
 __package__ = "archivebox.machine"
 
+import json
 import shlex
 from pathlib import Path
 
@@ -13,13 +14,50 @@ from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django_object_actions import action
 
-from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
-from archivebox.core.widgets import render_copy_block
+from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 from archivebox.machine.env_util import env_to_dotenv_text
 from archivebox.machine.models import Binary, Machine, NetworkInterface, Process
 from archivebox.misc.logging_util import printable_filesize
 
 
+def _render_copy_block(text: str, *, multiline: bool = False):
+    if multiline:
+        return format_html(
+            """
+            <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
+                <button type="button"
+                        data-command="{}"
+                        onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
+                        style="position: absolute; top: 6px; right: 6px; z-index: 1; padding: 2px 8px; border: 0; border-radius: 4px; background: #e2e8f0; color: #334155; font-size: 11px; cursor: pointer;">
+                    Copy
+                </button>
+                <pre title="{}" style="display: block; width: 100%; max-width: 100%; overflow: auto; max-height: 300px; margin: 0; padding: 8px 56px 8px 8px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; font-size: 11px; line-height: 1.45; white-space: pre-wrap; word-break: break-word; box-sizing: border-box;">{}</pre>
+            </div>
+            """,
+            text,
+            text,
+            text,
+        )
+    return format_html(
+        """
+        <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
+            <button type="button"
+                    data-command="{}"
+                    onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
+                    style="position: absolute; top: 6px; right: 6px; z-index: 1; padding: 2px 8px; border: 0; border-radius: 4px; background: #e2e8f0; color: #334155; font-size: 11px; cursor: pointer;">
+                Copy
+            </button>
+            <code title="{}" style="display: block; width: 100%; max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 8px 56px 8px 8px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; font-size: 11px; box-sizing: border-box;">
+                {}
+            </code>
+        </div>
+        """,
+        text,
+        text,
+        text,
+    )
+
+
 def _format_process_duration_seconds(started_at, ended_at) -> str:
     if not started_at:
         return "-"
@@ -84,27 +122,59 @@ class MachineAdmin(ConfigEditorMixin, BaseModelAdmin):
     readonly_fields = ("guid", "created_at", "modified_at", "ips")
 
     fieldsets = (
-        card_fieldset("Identity", ("hostname", "guid", "ips")),
-        card_fieldset("Hardware", ("hw_manufacturer", "hw_product", "hw_uuid", "hw_in_docker", "hw_in_vm")),
-        card_fieldset("Operating System", ("os_platform", "os_family", "os_arch", "os_kernel", "os_release")),
-        card_fieldset("Statistics", ("stats", "num_uses_succeeded", "num_uses_failed")),
-        card_fieldset(
+        (
+            "Identity",
+            {
+                "fields": ("hostname", "guid", "ips"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Hardware",
+            {
+                "fields": ("hw_manufacturer", "hw_product", "hw_uuid", "hw_in_docker", "hw_in_vm"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Operating System",
+            {
+                "fields": ("os_platform", "os_family", "os_arch", "os_kernel", "os_release"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Statistics",
+            {
+                "fields": ("stats", "num_uses_succeeded", "num_uses_failed"),
+                "classes": ("card",),
+            },
+        ),
+        (
             "Configuration",
-            ("config",),
-            wide=True,
-            description=mark_safe(
-                '<div style="padding:8px 10px;margin-bottom:8px;background:#fff7ed;'
-                "border:1px solid #fed7aa;border-left:4px solid #f59e0b;border-radius:4px;"
-                'color:#7c2d12;font-size:12px;line-height:1.45;">'
-                "<b>Heads up:</b> saving here also rewrites "
-                "<code>data/ArchiveBox.conf</code> on disk to match — the two stores are "
-                "kept in 1:1 sync, so any keys you remove here will be removed from the file "
-                "too. Edits to <code>ArchiveBox.conf</code> (or <code>archivebox config --set</code>) "
-                "propagate back into this field on the next request."
-                "</div>",
-            ),
+            {
+                "fields": ("config",),
+                "classes": ("card", "wide"),
+                "description": mark_safe(
+                    '<div style="padding:8px 10px;margin-bottom:8px;background:#fff7ed;'
+                    "border:1px solid #fed7aa;border-left:4px solid #f59e0b;border-radius:4px;"
+                    'color:#7c2d12;font-size:12px;line-height:1.45;">'
+                    "<b>Heads up:</b> saving here also rewrites "
+                    "<code>data/ArchiveBox.conf</code> on disk to match — the two stores are "
+                    "kept in 1:1 sync, so any keys you remove here will be removed from the file "
+                    "too. Edits to <code>ArchiveBox.conf</code> (or <code>archivebox config --set</code>) "
+                    "propagate back into this field on the next request."
+                    "</div>",
+                ),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": ("created_at", "modified_at"),
+                "classes": ("card",),
+            },
         ),
-        card_fieldset("Timestamps", ("created_at", "modified_at")),
     )
 
     list_filter = ("hw_in_docker", "hw_in_vm", "os_arch", "os_family", "os_platform")
@@ -120,6 +190,12 @@ class MachineAdmin(ConfigEditorMixin, BaseModelAdmin):
             ", ".join(machine.networkinterface_set.values_list("ip_public", flat=True)),
         )
 
+    @admin.display(description="Health", ordering="health")
+    def health_display(self, obj):
+        h = obj.health
+        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
+        return format_html('<span style="color: {};">{}</span>', color, h)
+
     @admin.display(description="ID", ordering="id")
     def id_display(self, machine):
         # Highlight the row representing the machine that ``Machine.current()``
@@ -199,11 +275,41 @@ class NetworkInterfaceAdmin(BaseModelAdmin):
     readonly_fields = ("machine", "created_at", "modified_at", "mac_address", "ip_public", "ip_local", "dns_server")
 
     fieldsets = (
-        card_fieldset("Machine", ("machine",)),
-        card_fieldset("Network", ("iface", "ip_public", "ip_local", "mac_address", "dns_server")),
-        card_fieldset("Location", ("hostname", "isp", "city", "region", "country")),
-        card_fieldset("Usage", ("num_uses_succeeded", "num_uses_failed")),
-        card_fieldset("Timestamps", ("created_at", "modified_at")),
+        (
+            "Machine",
+            {
+                "fields": ("machine",),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Network",
+            {
+                "fields": ("iface", "ip_public", "ip_local", "mac_address", "dns_server"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Location",
+            {
+                "fields": ("hostname", "isp", "city", "region", "country"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Usage",
+            {
+                "fields": ("num_uses_succeeded", "num_uses_failed"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": ("created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
     )
 
     list_filter = ("isp", "country", "region")
@@ -220,6 +326,12 @@ class NetworkInterfaceAdmin(BaseModelAdmin):
             iface.machine.hostname,
         )
 
+    @admin.display(description="Health", ordering="health")
+    def health_display(self, obj):
+        h = obj.health
+        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
+        return format_html('<span style="color: {};">{}</span>', color, h)
+
 
 class BinaryAdmin(BaseModelAdmin):
     list_display = ("id", "created_at", "machine_info", "name", "binprovider", "version", "abspath", "sha256", "status", "health_display")
@@ -229,12 +341,48 @@ class BinaryAdmin(BaseModelAdmin):
     readonly_fields = ("created_at", "modified_at", "output_dir")
 
     fieldsets = (
-        card_fieldset("Binary Info", ("name", "binproviders", "binprovider", "overrides")),
-        card_fieldset("Location", ("machine", "abspath")),
-        card_fieldset("Version", ("version", "sha256")),
-        card_fieldset("State", ("status", "retry_at", "output_dir")),
-        card_fieldset("Usage", ("num_uses_succeeded", "num_uses_failed")),
-        card_fieldset("Timestamps", ("created_at", "modified_at")),
+        (
+            "Binary Info",
+            {
+                "fields": ("name", "binproviders", "binprovider", "overrides"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Location",
+            {
+                "fields": ("machine", "abspath"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Version",
+            {
+                "fields": ("version", "sha256"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "State",
+            {
+                "fields": ("status", "retry_at", "output_dir"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Usage",
+            {
+                "fields": ("num_uses_succeeded", "num_uses_failed"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": ("created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
     )
 
     list_filter = ("name", "binprovider", "status", "machine_id")
@@ -251,6 +399,12 @@ class BinaryAdmin(BaseModelAdmin):
             binary.machine.hostname,
         )
 
+    @admin.display(description="Health", ordering="health")
+    def health_display(self, obj):
+        h = obj.health
+        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
+        return format_html('<span style="color: {};">{}</span>', color, h)
+
 
 class ProcessAdmin(BaseModelAdmin):
     list_display = (
@@ -309,10 +463,34 @@ class ProcessAdmin(BaseModelAdmin):
     )
 
     fieldsets = (
-        card_fieldset("Process Info", ("machine", "archiveresult_link", "snapshot_link", "crawl_link", "status", "retry_at")),
-        card_fieldset("Command", ("cmd_display", "pwd", "env_display", "timeout"), wide=True),
-        card_fieldset("Execution", ("binary_link", "iface_link", "pid", "exit_code", "url")),
-        card_fieldset("Timing", ("started_at", "ended_at", "duration_display")),
+        (
+            "Process Info",
+            {
+                "fields": ("machine", "archiveresult_link", "snapshot_link", "crawl_link", "status", "retry_at"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Command",
+            {
+                "fields": ("cmd_display", "pwd", "env_display", "timeout"),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Execution",
+            {
+                "fields": ("binary_link", "iface_link", "pid", "exit_code", "url"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timing",
+            {
+                "fields": ("started_at", "ended_at", "duration_display"),
+                "classes": ("card",),
+            },
+        ),
         (
             "Output",
             {
@@ -320,7 +498,13 @@ class ProcessAdmin(BaseModelAdmin):
                 "classes": ("card", "wide", "collapse"),
             },
         ),
-        card_fieldset("Timestamps", ("created_at", "modified_at")),
+        (
+            "Timestamps",
+            {
+                "fields": ("created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
     )
 
     list_filter = ("status", "exit_code", "machine_id")
@@ -519,9 +703,36 @@ class ProcessAdmin(BaseModelAdmin):
     @admin.display(description="Output", ordering="archiveresult__output_size")
     def output_summary(self, process):
         try:
-            file_count, total_bytes = process.archiveresult.output_file_stats()
+            output_files = process.archiveresult.output_files or {}
         except Process.archiveresult.RelatedObjectDoesNotExist:
-            file_count, total_bytes = 0, 0
+            output_files = {}
+
+        if isinstance(output_files, str):
+            try:
+                output_files = json.loads(output_files)
+            except (json.JSONDecodeError, TypeError, ValueError):
+                output_files = {}
+
+        file_count = 0
+        total_bytes = 0
+
+        if isinstance(output_files, dict):
+            file_count = len(output_files)
+            items = output_files.values()
+        elif isinstance(output_files, (list, tuple, set)):
+            file_count = len(output_files)
+            items = output_files
+        else:
+            items = ()
+
+        for metadata in items:
+            if not isinstance(metadata, dict):
+                continue
+            size = metadata.get("size", 0)
+            try:
+                total_bytes += int(size or 0)
+            except (TypeError, ValueError):
+                continue
 
         file_label = "file" if file_count == 1 else "files"
         return format_html(
@@ -539,26 +750,26 @@ class ProcessAdmin(BaseModelAdmin):
             cmd = shlex.join(str(arg) for arg in process.cmd)
         else:
             cmd = str(process.cmd)
-        return render_copy_block(cmd)
+        return _render_copy_block(cmd)
 
     @admin.display(description="Environment")
     def env_display(self, process):
         env_text = env_to_dotenv_text(process.env)
         if not env_text:
             return "-"
-        return render_copy_block(env_text, multiline=True)
+        return _render_copy_block(env_text, multiline=True)
 
     @admin.display(description="Stdout")
     def stdout_display(self, process):
         if not process.stdout:
             return "-"
-        return render_copy_block(process.stdout, multiline=True)
+        return _render_copy_block(process.stdout, multiline=True)
 
     @admin.display(description="Stderr")
     def stderr_display(self, process):
         if not process.stderr:
             return "-"
-        return render_copy_block(process.stderr, multiline=True)
+        return _render_copy_block(process.stderr, multiline=True)
 
     @admin.display(description="ArchiveResult Output")
     def archiveresult_output_display(self, process):
@@ -568,7 +779,7 @@ class ProcessAdmin(BaseModelAdmin):
             return "-"
         if not output:
             return "-"
-        return render_copy_block(output, multiline=True)
+        return _render_copy_block(output, multiline=True)
 
 
 def register_admin(admin_site):

archivebox/machine/detect.py

@@ -217,27 +217,26 @@ def get_isp_info(ip=None):
     }
 
 
-def get_local_interface(address: str) -> dict[str, str]:
-    """Describe the real host interface owning an IPv4 address."""
-    for interface, addresses in psutil.net_if_addrs().items():
-        if any(ip.address == address for ip in ip_addrs(addresses)):
-            return {
-                "iface": interface,
-                "mac_address": next((mac.address for mac in mac_addrs(addresses)), ""),
-                "ip_local": address,
-            }
-    raise ValueError(f"No host interface owns {address}")
-
-
 def get_host_network() -> dict[str, Any]:
-    interface = get_local_interface(get_local_ip())
-    public_ip = get_public_ip()
-    return {
-        "hostname": max([socket.gethostname(), platform.node()], key=len),
-        **interface,
-        "ip_public": public_ip,
-        **get_isp_info(public_ip),
-    }
+    default_gateway_local_ip = get_local_ip()
+    gateways = psutil.net_if_addrs()
+
+    for interface, ips in gateways.items():
+        for local_ip in ip_addrs(ips):
+            if default_gateway_local_ip == local_ip.address:
+                mac_address = next(mac_addrs(ips)).address
+                public_ip = get_public_ip()
+                return {
+                    "hostname": max([socket.gethostname(), platform.node()], key=len),
+                    "iface": interface,
+                    "mac_address": mac_address,
+                    "ip_local": local_ip.address,
+                    "ip_public": public_ip,
+                    # "is_behind_nat": local_ip.address != public_ip,
+                    **get_isp_info(public_ip),
+                }
+
+    raise Exception("Could not determine host network info")
 
 
 def get_os_info() -> dict[str, Any]:

archivebox/machine/models/__init__.py

@@ -1,50 +0,0 @@
-"""Machine, interface, binary, and process persistence with process-local caches."""
-
-from __future__ import annotations
-
-from .binaries import Binary as Binary
-from .binaries import BinaryManager as BinaryManager
-from .binaries import _canonical_binary_name as _canonical_binary_name
-from .binaries import _find_existing_binary_for_reference as _find_existing_binary_for_reference
-from .constants import (
-    BINARY_RECHECK_INTERVAL as BINARY_RECHECK_INTERVAL,
-)
-from .constants import (
-    MACHINE_RECHECK_INTERVAL as MACHINE_RECHECK_INTERVAL,
-)
-from .constants import (
-    NETWORK_INTERFACE_RECHECK_INTERVAL as NETWORK_INTERFACE_RECHECK_INTERVAL,
-)
-from .constants import (
-    PID_REUSE_WINDOW as PID_REUSE_WINDOW,
-)
-from .constants import (
-    PROCESS_PID_NAMESPACE_KEY as PROCESS_PID_NAMESPACE_KEY,
-)
-from .constants import (
-    PROCESS_RECHECK_INTERVAL as PROCESS_RECHECK_INTERVAL,
-)
-from .constants import (
-    PROCESS_TIMEOUT_GRACE as PROCESS_TIMEOUT_GRACE,
-)
-from .constants import (
-    PSUTIL_AVAILABLE as PSUTIL_AVAILABLE,
-)
-from .constants import (
-    START_TIME_TOLERANCE as START_TIME_TOLERANCE,
-)
-from .interfaces import NetworkInterface as NetworkInterface
-from .interfaces import NetworkInterfaceManager as NetworkInterfaceManager
-from .machines import Machine as Machine
-from .machines import MachineManager as MachineManager
-from .machines import _sanitize_machine_config as _sanitize_machine_config
-from .processes import Process as Process
-from .processes import ProcessManager as ProcessManager
-from .processes import _default_exit_code_for_unowned_process as _default_exit_code_for_unowned_process
-from .processes import _get_process_binary_env_keys as _get_process_binary_env_keys
-from .processes import get_current_pid_namespace as get_current_pid_namespace
-
-_CURRENT_MACHINE: Machine | None = None
-_CURRENT_INTERFACE: NetworkInterface | None = None
-_CURRENT_BINARIES: dict[str, Binary] = {}
-_CURRENT_PROCESS: Process | None = None

archivebox/machine/models/binaries.py

@@ -1,425 +0,0 @@
-from __future__ import annotations
-
-import sys
-import uuid
-from datetime import timedelta
-from pathlib import Path
-from typing import Any
-
-from django.db import models, transaction
-from django.utils import timezone
-from django.utils.functional import cached_property
-
-from archivebox.base_models.models import ModelWithHealthStats
-from archivebox.config import CONSTANTS
-from archivebox.config.common import rprint
-from archivebox.machine import models as state
-from archivebox.uuid_compat import CompactUUIDField, uuid7
-from archivebox.workers.models import ModelWithQueue
-
-from .constants import BINARY_RECHECK_INTERVAL
-from .machines import Machine
-
-
-def _find_existing_binary_for_reference(machine: Machine, reference: str) -> Binary | None:
-    reference = str(reference or "").strip()
-    if not reference:
-        return None
-
-    qs = Binary.objects.filter(machine=machine)
-
-    direct_match = qs.filter(abspath=reference).order_by("-modified_at").first()
-    if direct_match:
-        return direct_match
-
-    ref_name = Path(reference).name
-    if ref_name:
-        named_match = qs.filter(name=ref_name).order_by("-modified_at").first()
-        if named_match:
-            return named_match
-
-    return qs.filter(name=reference).order_by("-modified_at").first()
-
-
-def _canonical_binary_name(name: Any) -> str:
-    name = str(name or "").strip()
-    if "/" in name or "\\" in name or name.startswith("~"):
-        return Path(name).expanduser().name
-    return name
-
-
-class BinaryManager(models.Manager):
-    def get_from_db_or_cache(self, name: str, abspath: str = "", version: str = "", sha256: str = "", binprovider: str = "env") -> Binary:
-        """Get or create an Binary record from the database or cache."""
-        from archivebox.machine.models import Machine
-
-        cached = state._CURRENT_BINARIES.get(name)
-        if cached and timezone.now() < cached.modified_at + timedelta(seconds=BINARY_RECHECK_INTERVAL):
-            return cached
-        state._CURRENT_BINARIES[name], _ = self.update_or_create(
-            machine=Machine.current(),
-            name=name,
-            binprovider=binprovider,
-            version=version,
-            abspath=abspath,
-            sha256=sha256,
-        )
-        return state._CURRENT_BINARIES[name]
-
-    def get_valid_binary(self, name: str, machine: Machine | None = None) -> Binary | None:
-        """Get a valid Binary for the given name on the current machine, or None if not found."""
-        from archivebox.machine.models import Machine
-
-        machine = machine or Machine.current()
-        return (
-            self.filter(
-                machine=machine,
-                name__iexact=name,
-            )
-            .exclude(abspath="")
-            .exclude(abspath__isnull=True)
-            .order_by("-modified_at")
-            .first()
-        )
-
-
-class Binary(ModelWithHealthStats, ModelWithQueue):
-    """
-    Tracks a binary on a specific machine.
-
-    Simple queue lifecycle with 2 states:
-    - queued: Binary needs to be installed
-    - installed: Binary installed successfully (abspath, version, sha256 populated)
-
-    Installation is synchronous during queued→installed transition.
-    If installation fails, Binary stays in queued with retry_at set for later retry.
-
-    BinaryService claims queued rows with a conditional update, then run()
-    emits an abxpkg BinaryRequestEvent and persists the resolved installation.
-    The database row is the only lifecycle state; there is intentionally no
-    second in-memory state machine to reconcile after a worker interruption.
-    """
-
-    class StatusChoices(models.TextChoices):
-        QUEUED = "queued", "Queued"
-        INSTALLED = "installed", "Installed"
-
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
-    machine = models.ForeignKey(Machine, on_delete=models.CASCADE, null=False)
-
-    # Binary metadata
-    name = models.CharField(max_length=63, default="", null=False, blank=True, db_index=True)
-    binproviders = models.CharField(
-        max_length=127,
-        default="env",
-        null=False,
-        blank=True,
-        help_text="Comma-separated list of allowed providers: apt,brew,pip,npm,env",
-    )
-    overrides = models.JSONField(
-        default=dict,
-        blank=True,
-        help_text="Provider-specific overrides: {'apt': {'install_args': ['pkg']}, ...}",
-    )
-
-    # Installation results (populated after installation)
-    binprovider = models.CharField(
-        max_length=31,
-        default="",
-        null=False,
-        blank=True,
-        help_text="Provider that successfully installed this binary",
-    )
-    abspath = models.CharField(max_length=255, default="", null=False, blank=True)
-    version = models.CharField(max_length=32, default="", null=False, blank=True)
-    sha256 = models.CharField(max_length=64, default="", null=False, blank=True)
-
-    # Durable queue lifecycle fields
-    status = ModelWithQueue.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED, max_length=16)
-    retry_at = ModelWithQueue.RetryAtField(
-        default=timezone.now,
-        help_text="When to retry this binary installation",
-    )
-
-    # Health stats
-    num_uses_failed = models.PositiveIntegerField(default=0)
-    num_uses_succeeded = models.PositiveIntegerField(default=0)
-
-    machine_id: uuid.UUID
-
-    INITIAL_STATE = StatusChoices.QUEUED
-    ACTIVE_STATE = StatusChoices.QUEUED
-    FINAL_STATES = (StatusChoices.INSTALLED,)
-    FINAL_OR_ACTIVE_STATES = (*FINAL_STATES, ACTIVE_STATE)
-    active_state: str = StatusChoices.QUEUED
-    warn_on_save_outside_runner = False
-
-    objects = BinaryManager()  # pyright: ignore[reportIncompatibleVariableOverride]
-
-    class Meta(ModelWithHealthStats.Meta, ModelWithQueue.Meta):
-        app_label = "machine"
-        verbose_name = "Binary"
-        verbose_name_plural = "Binaries"
-        unique_together = (("machine", "name", "abspath", "version", "sha256"),)
-
-    def __str__(self) -> str:
-        return f"{self.name}@{self.binprovider}+{self.abspath}@{self.version}"
-
-    @property
-    def is_valid(self) -> bool:
-        """A binary is valid if it has a resolved path and is marked installed."""
-        return bool(self.abspath) and self.status == self.StatusChoices.INSTALLED
-
-    @property
-    def can_install(self) -> bool:
-        return bool(self.name and self.binproviders)
-
-    @cached_property
-    def binary_info(self) -> dict:
-        """Return info about the binary."""
-        return {
-            "name": self.name,
-            "abspath": self.abspath,
-            "version": self.version,
-            "binprovider": self.binprovider,
-            "is_valid": self.is_valid,
-        }
-
-    @property
-    def output_dir(self) -> Path:
-        """
-        Get output directory for this binary's hook logs.
-        Path: data/machines/{machine_uuid}/binaries/{binary_name}/{binary_uuid}
-        """
-        return CONSTANTS.DATA_DIR / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)
-
-    def to_json(self) -> dict:
-        """
-        Convert Binary model instance to a JSON-serializable dict.
-        """
-        from archivebox.config import VERSION
-
-        is_installed = bool(self.abspath and self.version)
-        return {
-            "type": "Binary" if is_installed else "BinaryRequest",
-            "schema_version": VERSION,
-            "id": str(self.id),
-            "machine_id": str(self.machine_id),
-            "name": self.name,
-            "binproviders": self.binproviders,
-            "overrides": self.overrides,
-            "binprovider": self.binprovider,
-            "abspath": self.abspath,
-            "version": self.version,
-            "sha256": self.sha256,
-            "status": self.status,
-        }
-
-    @staticmethod
-    def from_json(record: dict[str, Any], overrides: dict[str, Any] | None = None):
-        """
-        Create/update Binary from JSON dict.
-
-        Handles two cases:
-        1. From binaries.json: creates queued binary with name, binproviders, overrides
-        2. From hook output: updates binary with abspath, version, sha256, binprovider
-
-        Args:
-            record: JSON dict with 'name' and either:
-                    - 'binproviders', 'overrides' (from binaries.json)
-                    - 'abspath', 'version', 'sha256', 'binprovider' (from hook output)
-            overrides: Not used
-
-        Returns:
-            Binary instance or None
-        """
-        from archivebox.machine.models import Machine
-
-        name = _canonical_binary_name(record.get("name"))
-        if not name:
-            return None
-
-        abspath, version = record.get("abspath"), record.get("version")
-        binproviders = record.get("binproviders")
-        installed = bool(abspath and version) and (bool(binproviders) or "binproviders" not in record)
-        if installed:
-            defaults = {
-                "abspath": abspath,
-                "version": version,
-                "sha256": record.get("sha256", ""),
-                "binprovider": record.get("binprovider", "env"),
-                "status": Binary.StatusChoices.INSTALLED,
-                "retry_at": None,
-            }
-            if binproviders:
-                defaults["binproviders"] = binproviders
-        elif "binproviders" in record or ("overrides" in record and not abspath):
-            binary_overrides = record.get("overrides", {})
-            defaults = {
-                "binproviders": record.get("binproviders", "env"),
-                "overrides": binary_overrides if isinstance(binary_overrides, dict) else {},
-                "status": Binary.StatusChoices.QUEUED,
-                "retry_at": timezone.now(),
-            }
-        else:
-            return None
-
-        binary, _ = Binary.objects.update_or_create(machine=Machine.current(), name=name, defaults=defaults)
-        if installed:
-            from archivebox.config.common import get_config
-
-            binary.symlink_to_lib_bin_after_commit(get_config().ABXPKG_LIB_DIR / "bin")
-        return binary
-
-    def run(self):
-        """
-        Execute binary installation through the ArchiveBox binary runner.
-        """
-        from archivebox.services.runner import run_binary
-
-        run_binary(str(self.id))
-
-    def install(self) -> bool:
-        """Run one synchronous installation attempt for a claimed Binary."""
-        if self.status == self.StatusChoices.INSTALLED:
-            return True
-        if not self.can_install:
-            return False
-
-        rprint(f"[cyan]      🔄 installing {self.name}[/cyan]", file=sys.stderr)
-        self.run()
-        self.refresh_from_db()
-        if self.status != self.StatusChoices.INSTALLED:
-            self.update_and_requeue(
-                retry_at=timezone.now() + timedelta(seconds=300),
-                status=self.StatusChoices.QUEUED,
-            )
-            self.increment_health_stats(success=False)
-            raise RuntimeError(f"Binary {self.name} installation failed")
-
-        self.update_and_requeue(retry_at=None, status=self.StatusChoices.INSTALLED)
-        self.increment_health_stats(success=True)
-        return True
-
-    def advance_lifecycle(self) -> bool:
-        """Advance the explicit binary lifecycle after its queue row is claimed."""
-        return self.install()
-
-    def install_claimed(self, *, lock_seconds: int = 600) -> bool:
-        if not self.claim_processing_lock(lock_seconds=lock_seconds):
-            return False
-        self.refresh_from_db()
-        return self.advance_lifecycle()
-
-    def cleanup(self):
-        """
-        Clean up background binary installation hooks.
-
-        Called after an installation attempt if needed (not typically used for binaries
-        since installations are foreground, but included for consistency).
-        """
-
-        # Clean up .pid files from output directory
-        output_dir = self.output_dir
-        if output_dir.exists():
-            for pid_file in output_dir.glob("**/*.pid"):
-                pid_file.unlink(missing_ok=True)
-
-    def symlink_to_lib_bin(self, lib_bin_dir: str | Path) -> Path | None:
-        """
-        Symlink this binary into a derived lib/bin directory for human-facing convenience.
-
-        After a binary is installed by any binprovider (pip, npm, brew, apt, etc),
-        we can optionally expose a flat convenience directory for shell users.
-        ArchiveBox/abx-dl runtime lookup must use the provider-specific ABXPKG_LIB_DIR
-        paths, not this indirection.
-
-        Args:
-            lib_bin_dir: Path to the derived convenience bin dir (e.g., /data/lib/bin)
-
-        Returns:
-            Path to the created symlink, or None if symlinking failed
-
-        Example:
-            >>> binary = Binary.objects.get(name='yt-dlp')
-            >>> binary.symlink_to_lib_bin('/data/lib/arm64-darwin/bin')
-            Path('/data/lib/arm64-darwin/bin/yt-dlp')
-        """
-        import sys
-        from pathlib import Path
-
-        if not self.abspath:
-            return None
-
-        binary_abspath = Path(self.abspath).resolve()
-        lib_bin_dir = Path(lib_bin_dir).resolve()
-        binary_parts = binary_abspath.parts
-        try:
-            app_index = next(index for index, part in enumerate(binary_parts) if part.endswith(".app"))
-        except StopIteration:
-            app_index = -1
-
-        # Create the derived convenience bin dir if it doesn't exist.
-        try:
-            lib_bin_dir.mkdir(parents=True, exist_ok=True)
-        except (OSError, PermissionError) as e:
-            print(f"Failed to create lib/bin convenience dir {lib_bin_dir}: {e}", file=sys.stderr)
-            return None
-
-        # Expose the canonical Binary.name in the convenience bin dir. Some providers point
-        # abspath at implementation files like cli.js or manifest.json; those
-        # are valid targets, but they are not user-facing binary names.
-        binary_name = _canonical_binary_name(self.name) or binary_abspath.name
-        symlink_path = lib_bin_dir / binary_name
-
-        if app_index != -1 and len(binary_parts) > app_index + 2 and binary_parts[app_index + 1 : app_index + 3] == ("Contents", "MacOS"):
-            if symlink_path.exists() or symlink_path.is_symlink():
-                try:
-                    symlink_path.unlink()
-                except (OSError, PermissionError) as e:
-                    print(f"Failed to remove existing file at {symlink_path}: {e}", file=sys.stderr)
-                    return None
-            return binary_abspath
-
-        # Remove existing symlink/file if it exists
-        if symlink_path.exists() or symlink_path.is_symlink():
-            try:
-                # Check if it's already pointing to the right place
-                if symlink_path.is_symlink() and symlink_path.resolve() == binary_abspath:
-                    # Already correctly symlinked, nothing to do
-                    return symlink_path
-
-                # Remove old symlink/file
-                symlink_path.unlink()
-            except (OSError, PermissionError) as e:
-                print(f"Failed to remove existing file at {symlink_path}: {e}", file=sys.stderr)
-                return None
-
-        # Create new symlink
-        try:
-            symlink_path.symlink_to(binary_abspath)
-            return symlink_path
-        except (OSError, PermissionError) as e:
-            print(f"Failed to create symlink {symlink_path} -> {binary_abspath}: {e}", file=sys.stderr)
-            return None
-
-    def symlink_to_lib_bin_after_commit(self, lib_bin_dir: str | Path) -> None:
-        """
-        Symlink after the current DB transaction commits.
-
-        Binary rows are projections of provider/hook state and are allowed to be
-        updated directly, but filesystem writes must not run while an outer
-        transaction is still open. Refetch after commit so the symlink points at
-        the committed row, not a possibly-rolled-back in-memory value.
-        """
-        binary_id = self.id
-        lib_bin_path = Path(lib_bin_dir)
-
-        def create_symlink() -> None:
-            binary = type(self).objects.filter(id=binary_id).first()
-            if binary is not None:
-                binary.symlink_to_lib_bin(lib_bin_path)
-
-        transaction.on_commit(create_symlink)

archivebox/machine/models/constants.py

@@ -1,26 +0,0 @@
-from datetime import timedelta
-from typing import TYPE_CHECKING, Any, cast
-
-_psutil: Any | None = None
-try:
-    import psutil as _psutil_import
-
-    PSUTIL_AVAILABLE = True
-except ImportError:
-    PSUTIL_AVAILABLE = False
-else:
-    _psutil = _psutil_import
-
-if TYPE_CHECKING:
-    import psutil
-else:
-    psutil = cast(Any, _psutil)
-
-MACHINE_RECHECK_INTERVAL = 7 * 24 * 60 * 60
-NETWORK_INTERFACE_RECHECK_INTERVAL = 1 * 60 * 60
-BINARY_RECHECK_INTERVAL = 1 * 30 * 60
-PROCESS_RECHECK_INTERVAL = 60  # Re-validate every 60 seconds
-PID_REUSE_WINDOW = timedelta(hours=24)  # Max age for considering a PID match valid
-PROCESS_TIMEOUT_GRACE = timedelta(seconds=30)  # Extra margin before force-cleaning timed-out RUNNING rows
-START_TIME_TOLERANCE = 5.0  # Seconds tolerance for start time matching
-PROCESS_PID_NAMESPACE_KEY = "_ARCHIVEBOX_PID_NAMESPACE"

archivebox/machine/models/interfaces.py

@@ -1,102 +0,0 @@
-from __future__ import annotations
-
-import uuid
-from datetime import timedelta
-
-from django.db import IntegrityError, models
-from django.utils import timezone
-
-from archivebox.base_models.models import ModelWithHealthStats
-from archivebox.machine import models as state
-from archivebox.uuid_compat import CompactUUIDField, uuid7
-
-from ..detect import get_host_network
-from .constants import NETWORK_INTERFACE_RECHECK_INTERVAL
-from .machines import Machine
-
-
-class NetworkInterfaceManager(models.Manager):
-    def current(self) -> NetworkInterface:
-        return NetworkInterface.current()
-
-
-class NetworkInterface(ModelWithHealthStats):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
-    machine = models.ForeignKey(Machine, on_delete=models.CASCADE, default=None, null=False)
-    mac_address = models.CharField(max_length=17, default=None, null=False, editable=False)
-    ip_public = models.GenericIPAddressField(default=None, null=False, editable=False)
-    ip_local = models.GenericIPAddressField(default=None, null=False, editable=False)
-    dns_server = models.GenericIPAddressField(default=None, null=False, editable=False)
-    hostname = models.CharField(max_length=63, default="", null=False)
-    iface = models.CharField(max_length=15, default="", null=False)
-    isp = models.CharField(max_length=63, default="", null=False)
-    city = models.CharField(max_length=63, default="", null=False)
-    region = models.CharField(max_length=63, default="", null=False)
-    country = models.CharField(max_length=63, default="", null=False)
-    objects = NetworkInterfaceManager()  # pyright: ignore[reportIncompatibleVariableOverride]
-    machine_id: uuid.UUID
-
-    class Meta(ModelWithHealthStats.Meta):
-        app_label = "machine"
-        unique_together = (("machine", "ip_public", "ip_local", "dns_server"),)
-        constraints = [
-            models.UniqueConstraint(
-                fields=["machine", "ip_public", "ip_local", "dns_server"],
-                name="unique_network_interface_identity",
-            ),
-        ]
-
-    @classmethod
-    def current(cls, refresh: bool = False) -> NetworkInterface:
-        from archivebox.machine.models import Machine
-
-        machine = Machine.current(refresh=refresh)
-        if state._CURRENT_INTERFACE and state._CURRENT_INTERFACE.machine_id == machine.id:
-            if not refresh:
-                # Callers that pass refresh=False are asking for attribution to
-                # the currently known interface, not for public-IP/ISP probing.
-                # Maintenance paths create many short-lived services per run;
-                # expiring this in-memory object by age forced every crawl to
-                # hit external network APIs even though Process rows only need
-                # a stable existing FK. Active downloading paths opt into live
-                # detection with refresh=True.
-                return state._CURRENT_INTERFACE
-            if timezone.now() < state._CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL):
-                return state._CURRENT_INTERFACE
-        state._CURRENT_INTERFACE = None
-
-        if not refresh:
-            state._CURRENT_INTERFACE = cls.objects.filter(machine=machine).order_by("-modified_at", "-created_at").first()
-            if state._CURRENT_INTERFACE is not None:
-                return state._CURRENT_INTERFACE
-
-        net_info = get_host_network()
-        lookup = dict(
-            machine=machine,
-            ip_public=net_info.pop("ip_public"),
-            ip_local=net_info.pop("ip_local"),
-            dns_server=net_info.pop("dns_server"),
-        )
-        state._CURRENT_INTERFACE = cls.objects.filter(**lookup).order_by("-modified_at", "-created_at").first()
-        if state._CURRENT_INTERFACE is None:
-            try:
-                state._CURRENT_INTERFACE = cls.objects.create(**lookup, **net_info)
-            except IntegrityError:
-                state._CURRENT_INTERFACE = cls.objects.filter(**lookup).order_by("-modified_at", "-created_at").first()
-                if state._CURRENT_INTERFACE is None:
-                    raise
-        else:
-            # Avoid update_or_create() here: command startup calls this before
-            # leadership handoff, and SQLite should not take a write lock unless
-            # the cached interface metadata is actually stale.
-            if refresh or timezone.now() >= state._CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL):
-                updates = ["modified_at"]
-                for key, value in net_info.items():
-                    if state._CURRENT_INTERFACE.__dict__.get(key) != value:
-                        setattr(state._CURRENT_INTERFACE, key, value)
-                        updates.append(key)
-                if len(updates) > 1:
-                    state._CURRENT_INTERFACE.save(update_fields=updates)
-        return state._CURRENT_INTERFACE

archivebox/machine/models/machines.py

@@ -1,281 +0,0 @@
-from __future__ import annotations
-
-import socket
-from datetime import timedelta
-from pathlib import Path
-from typing import TYPE_CHECKING, Any
-
-from django.db import models
-from django.utils import timezone
-
-from archivebox.base_models.models import ModelWithHealthStats, normalize_config_json_values
-from archivebox.machine import models as state
-from archivebox.uuid_compat import CompactUUIDField, uuid7
-
-from ..detect import get_host_guid, get_host_stats, get_os_info, get_vm_info
-from .constants import MACHINE_RECHECK_INTERVAL
-
-if TYPE_CHECKING:
-    from .interfaces import NetworkInterface
-
-
-def _sanitize_machine_config(config: dict[str, Any] | None, *, lib_dir: str | Path | None = None) -> dict[str, Any]:
-    """Validate ``Machine.config`` in place.
-
-    Drops stale ``*_BINARY`` overrides whose path no longer exists or whose
-    path falls outside of ``ABXPKG_LIB_DIR`` (so a binary uninstall or a lib_dir
-    move clears the override automatically). Non-``_BINARY`` keys
-    (``BASE_URL``, ``SERVER_SECURITY_MODE``, plugin tunables, etc.) are
-    pass-through — they're arbitrary config overrides and not ours to filter.
-    """
-    if not isinstance(config, dict):
-        return {}
-
-    sanitized = dict(config)
-    active_lib_dir = Path(lib_dir).expanduser().absolute() if lib_dir else None
-    for key, value in list(sanitized.items()):
-        if not str(key).endswith("_BINARY"):
-            continue
-        if not isinstance(value, str):
-            continue
-        value = value.strip()
-        if not value:
-            sanitized.pop(key, None)
-            continue
-        if "/" in value or value.startswith("~"):
-            try:
-                path = Path(value).expanduser()
-                if not path.exists():
-                    sanitized.pop(key, None)
-                    continue
-                if active_lib_dir is not None:
-                    resolved_path = path.absolute()
-                    try:
-                        resolved_path.relative_to(active_lib_dir)
-                    except ValueError:
-                        sanitized.pop(key, None)
-            except OSError:
-                sanitized.pop(key, None)
-    return sanitized
-
-
-class MachineManager(models.Manager):
-    def current(self) -> Machine:
-        return Machine.current()
-
-
-class Machine(ModelWithHealthStats):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
-    guid = models.CharField(max_length=64, default=None, null=False, unique=True, editable=False)
-    hostname = models.CharField(max_length=63, default=None, null=False)
-    hw_in_docker = models.BooleanField(default=False, null=False)
-    hw_in_vm = models.BooleanField(default=False, null=False)
-    hw_manufacturer = models.CharField(max_length=63, default=None, null=False)
-    hw_product = models.CharField(max_length=63, default=None, null=False)
-    hw_uuid = models.CharField(max_length=255, default=None, null=False)
-    os_arch = models.CharField(max_length=15, default=None, null=False)
-    os_family = models.CharField(max_length=15, default=None, null=False)
-    os_platform = models.CharField(max_length=63, default=None, null=False)
-    os_release = models.CharField(max_length=63, default=None, null=False)
-    os_kernel = models.CharField(max_length=255, default=None, null=False)
-    stats = models.JSONField(default=dict, null=True, blank=True)
-    config = models.JSONField(
-        default=dict,
-        null=True,
-        blank=True,
-        help_text="Machine-specific config overrides.",
-    )
-    num_uses_failed = models.PositiveIntegerField(default=0)
-    num_uses_succeeded = models.PositiveIntegerField(default=0)
-
-    objects = MachineManager()  # pyright: ignore[reportIncompatibleVariableOverride]
-    networkinterface_set: models.Manager[NetworkInterface]
-
-    class Meta(ModelWithHealthStats.Meta):
-        app_label = "machine"
-
-    @classmethod
-    def current(cls, refresh: bool = False) -> Machine:
-        if refresh:
-            state._CURRENT_MACHINE = None
-        if state._CURRENT_MACHINE:
-            if timezone.now() < state._CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
-                # Reconciliation can save() and clear the shared cache; keep
-                # this call's instance so we never return None after that save.
-                machine = state._CURRENT_MACHINE
-                # One-time-per-process reconciliation between ArchiveBox.conf
-                # and Machine.config. Fast-path: bool check + early-return when
-                # the sync has already run, so the cached-machine return path
-                # stays sub-microsecond.
-                try:
-                    from archivebox.config.collection import sync_machine_and_file
-
-                    sync_machine_and_file(machine)
-                except Exception:
-                    pass
-                return machine
-            else:
-                state._CURRENT_MACHINE = None
-
-        host_guid = get_host_guid()
-        # save() intentionally invalidates state._CURRENT_MACHINE so config edits
-        # made through other instances become visible in this process. Host
-        # refresh (after seven days), sanitation, and reconciliation can all
-        # save here, so the working instance must not live only in that cache.
-        # Otherwise a routine refresh leaves startup dereferencing None.
-        try:
-            machine = cls.objects.get(guid=host_guid)
-        except cls.DoesNotExist:
-            config = {}
-            try:
-                from archivebox.config.collection import _coerce_from_str_dict, _load_file_config_dict
-
-                file_config, _file_mtime = _load_file_config_dict()
-                config = _coerce_from_str_dict(file_config)
-            except Exception:
-                config = {}
-            machine = cls.objects.create(
-                guid=host_guid,
-                hostname=socket.gethostname(),
-                config=config,
-                **get_os_info(),
-                **get_vm_info(),
-                stats=get_host_stats(),
-            )
-        else:
-            if timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
-                for key, value in {
-                    "hostname": socket.gethostname(),
-                    **get_os_info(),
-                    **get_vm_info(),
-                    "stats": get_host_stats(),
-                }.items():
-                    setattr(machine, key, value)
-                machine.save(
-                    update_fields=[
-                        "hostname",
-                        "hw_in_docker",
-                        "hw_in_vm",
-                        "hw_manufacturer",
-                        "hw_product",
-                        "hw_uuid",
-                        "os_arch",
-                        "os_family",
-                        "os_platform",
-                        "os_release",
-                        "os_kernel",
-                        "stats",
-                        "modified_at",
-                    ],
-                )
-        machine = cls._sanitize_config(machine)
-        # Same one-time sync as the cached-return path. Triggers here on the
-        # very first ``Machine.current()`` call in a process before any
-        # cached return can occur.
-        try:
-            from archivebox.config.collection import sync_machine_and_file
-
-            sync_machine_and_file(machine)
-        except Exception:
-            pass
-        # Publish only after all save-capable steps; publishing earlier would
-        # let those steps invalidate the cache we are in the middle of filling.
-        state._CURRENT_MACHINE = machine
-        return machine
-
-    @classmethod
-    def _sanitize_config(cls, machine: Machine) -> Machine:
-        from archivebox.config.common import get_config
-
-        sanitized = _sanitize_machine_config(machine.config, lib_dir=get_config(include_machine=False).ABXPKG_LIB_DIR)
-        current = machine.config or {}
-        if sanitized != current:
-            machine.config = sanitized
-            machine.save(update_fields=["config", "modified_at"])
-        return machine
-
-    def to_json(self) -> dict:
-        """
-        Convert Machine model instance to a JSON-serializable dict.
-        """
-        from archivebox.config import VERSION
-        from archivebox.config.common import redact_sensitive_config
-
-        return {
-            "type": "Machine",
-            "schema_version": VERSION,
-            "id": str(self.id),
-            "guid": self.guid,
-            "hostname": self.hostname,
-            "hw_in_docker": self.hw_in_docker,
-            "hw_in_vm": self.hw_in_vm,
-            "hw_manufacturer": self.hw_manufacturer,
-            "hw_product": self.hw_product,
-            "hw_uuid": self.hw_uuid,
-            "os_arch": self.os_arch,
-            "os_family": self.os_family,
-            "os_platform": self.os_platform,
-            "os_kernel": self.os_kernel,
-            "os_release": self.os_release,
-            "stats": self.stats,
-            "config": redact_sensitive_config(self.config),
-        }
-
-    @staticmethod
-    def from_json(record: dict[str, Any], overrides: dict[str, Any] | None = None):
-        """
-        Update Machine config from JSON dict.
-
-        Args:
-            record: JSON dict with 'config': {key: value} patch
-            overrides: Not used
-
-        Returns:
-            Machine instance or None
-        """
-        config_patch = _sanitize_machine_config(record.get("config"))
-        if config_patch:
-            machine = Machine.current()
-            machine.config = _sanitize_machine_config(machine.config)
-            machine.config.update(config_patch)
-            machine.save(update_fields=["config"])
-            return machine
-        return None
-
-    def save(self, *args, **kwargs):
-        normalized_config = normalize_config_json_values(self.config)
-        if normalized_config != self.config:
-            self.config = normalized_config
-            update_fields = kwargs.get("update_fields")
-            if update_fields is not None:
-                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
-
-        # Drop the ``Machine.current()`` module-level cache on every save so
-        # config edits (admin form, from_json, etc.) become live in the same
-        # process without waiting out the 7-day ``MACHINE_RECHECK_INTERVAL``.
-        # The cache reads ``state._CURRENT_MACHINE.modified_at`` and that value
-        # never moves forward on the cached object even when the row is
-        # updated in the DB, so without this we'd keep serving stale
-        # ``machine.config`` (incl. ``BASE_URL``) until the worker restarts.
-        update_fields = kwargs.get("update_fields")
-        super().save(*args, **kwargs)
-        if state._CURRENT_MACHINE is not None and state._CURRENT_MACHINE.pk == self.pk:
-            state._CURRENT_MACHINE = None
-
-        # Mirror Machine.config into ArchiveBox.conf so the two stores stay
-        # 1:1. Skipped when ``update_fields`` is set and doesn't touch
-        # ``config`` (binary autodetection + ``hostname``/``stats`` refreshes
-        # save fields we don't need to disk-mirror, which keeps hot paths
-        # zero-IO). Errors during mirroring are swallowed: the DB write
-        # already succeeded and we don't want a config-file write hiccup to
-        # turn a routine save into a 500.
-        if update_fields is not None and "config" not in update_fields:
-            return
-        try:
-            from archivebox.config.collection import mirror_machine_config_to_file
-
-            mirror_machine_config_to_file(self.config)
-        except Exception:
-            pass

archivebox/misc/checks.py

@@ -406,86 +406,3 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
                 f"ABXPKG_LIB_DIR={lib_dir} is invalid, ArchiveBox is unable to use it and dependencies will fail to install.",
             ) from e
     return False
-
-
-MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES = 256 * 1024 * 1024
-
-
-MIN_DEPENDENCY_INSTALL_AVAILABLE_MEMORY_BYTES = MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES
-
-
-MIN_CRAWL_AVAILABLE_MEMORY_BYTES = 512 * 1024 * 1024
-
-
-def _read_cgroup_limit(path: Path) -> int | None:
-    try:
-        raw_value = path.read_text().strip()
-        return None if raw_value == "max" else int(raw_value)
-    except (OSError, ValueError):
-        return None
-
-
-def effective_available_memory_bytes() -> int:
-    import psutil
-
-    available = psutil.virtual_memory().available + psutil.swap_memory().free
-
-    cgroup_root = Path("/sys/fs/cgroup")
-    memory_max = _read_cgroup_limit(cgroup_root / "memory.max")
-    memory_current = _read_cgroup_limit(cgroup_root / "memory.current")
-    swap_max = _read_cgroup_limit(cgroup_root / "memory.swap.max")
-    swap_current = _read_cgroup_limit(cgroup_root / "memory.swap.current")
-    if memory_max is not None and memory_current is not None:
-        cgroup_available = max(memory_max - memory_current, 0)
-        if swap_max is not None and swap_current is not None:
-            cgroup_available += max(swap_max - swap_current, 0)
-        available = min(available, cgroup_available)
-
-    return available
-
-
-def _require_available_memory(operation: str, idle_message: str, available_bytes: int, required_bytes: int) -> None:
-    from archivebox.misc.logging import STDERR
-
-    if available_bytes >= required_bytes:
-        return
-
-    available_mib = available_bytes // (1024 * 1024)
-    required_mib = required_bytes // (1024 * 1024)
-    STDERR.print(f"[red][X] Not enough available memory to {operation} safely.[/red]")
-    STDERR.print(
-        f"    Available RAM + swap: {available_mib} MiB; at least {required_mib} MiB must be free before this operation.",
-    )
-    STDERR.print(idle_message)
-    STDERR.print("    Use a host/container with at least 1 GB RAM or configure swap, then run the same command again.")
-    raise SystemExit(1)
-
-
-def require_server_worker_memory(available_bytes: int | None = None) -> None:
-    available_bytes = effective_available_memory_bytes() if available_bytes is None else available_bytes
-    _require_available_memory(
-        "start ArchiveBox",
-        "    No server, runner, or Sonic workers were started.",
-        available_bytes,
-        MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES,
-    )
-
-
-def require_dependency_install_memory(available_bytes: int | None = None) -> None:
-    available_bytes = effective_available_memory_bytes() if available_bytes is None else available_bytes
-    _require_available_memory(
-        "install ArchiveBox dependencies",
-        "    No plugin dependency installers were started.",
-        available_bytes,
-        MIN_DEPENDENCY_INSTALL_AVAILABLE_MEMORY_BYTES,
-    )
-
-
-def require_crawl_memory(available_bytes: int | None = None) -> None:
-    available_bytes = effective_available_memory_bytes() if available_bytes is None else available_bytes
-    _require_available_memory(
-        "archive a crawl",
-        "    No crawl, runner, or Sonic workers were started.",
-        available_bytes,
-        MIN_CRAWL_AVAILABLE_MEMORY_BYTES,
-    )

archivebox/misc/db.py

@@ -350,9 +350,7 @@ def sqlite_lock_holders(db_path: Path = CONSTANTS.DATABASE_FILE) -> list[str]:
     for proc in psutil.process_iter(["pid", "ppid", "name", "cmdline", "status"]):
         try:
             open_files = proc.open_files()
-        except (psutil.AccessDenied, psutil.NoSuchProcess, psutil.ZombieProcess, RuntimeError):
-            # macOS libproc can report an uninspectable process as RuntimeError.
-            # Optional holder diagnostics must not abort the database operation.
+        except (psutil.AccessDenied, psutil.NoSuchProcess, psutil.ZombieProcess):
             continue
         for open_file in open_files:
             try:
@@ -664,24 +662,3 @@ def apply_migrations(
         out1 = retry_sqlite_locks(migrate, label="applying migrations")
 
         return [line.strip() for line in out1.readlines() if line.strip()]
-
-
-def uuid_ref_query(field_name: str, ref: str):
-    from uuid import UUID
-    from django.db.models import Q
-
-    raw_ref = str(ref or "").strip()
-    query = Q(**{f"{field_name}__startswith": raw_ref})
-    if raw_ref:
-        query |= Q(**{f"{field_name}__icontains": raw_ref})
-    try:
-        parsed_uuid = UUID(raw_ref)
-    except (TypeError, ValueError):
-        normalized_ref = raw_ref.replace("-", "")
-        if normalized_ref and normalized_ref != raw_ref:
-            query |= Q(**{f"{field_name}__startswith": normalized_ref})
-            query |= Q(**{f"{field_name}__icontains": normalized_ref})
-    else:
-        query |= Q(**{field_name: parsed_uuid})
-        query |= Q(**{f"{field_name}__startswith": parsed_uuid.hex})
-    return query

archivebox/misc/hashing.py

@@ -0,0 +1,260 @@
+# Bootable hashing helpers (file content + mtime/size cache, mime detection).
+# MUST NOT import archivebox.config, archivebox.core, or Django — stdlib only.
+
+import hashlib
+import mimetypes
+from functools import lru_cache
+from pathlib import Path
+from collections.abc import Callable
+from datetime import datetime
+
+
+@lru_cache(maxsize=1024)
+def _cached_file_hash(filepath: str, size: int, mtime: float) -> str:
+    """Internal function to calculate file hash with cache key based on path, size and mtime."""
+    sha256_hash = hashlib.sha256()
+
+    with open(filepath, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            sha256_hash.update(chunk)
+
+    return sha256_hash.hexdigest()
+
+
+@lru_cache(maxsize=10)
+def hash_file(file_path: Path, pwd: Path | None = None) -> str:
+    """Calculate SHA256 hash of a file with caching based on path, size and mtime."""
+    pwd = Path(pwd) if pwd else None
+    file_path = Path(file_path)
+    if not file_path.is_absolute():
+        file_path = pwd / file_path if pwd else file_path.absolute()
+
+    abs_path = file_path.resolve()
+    stat_info = abs_path.stat()
+
+    return _cached_file_hash(
+        str(abs_path),
+        stat_info.st_size,
+        stat_info.st_mtime,
+    )
+
+
+@lru_cache(maxsize=10)
+def get_dir_hashes(dir_path: Path, pwd: Path | None = None, filter_func: Callable | None = None, max_depth: int = -1) -> dict[str, str]:
+    """Calculate SHA256 hashes for all files and directories recursively."""
+    pwd = Path(pwd) if pwd else None
+    dir_path = Path(dir_path)
+    if not dir_path.is_absolute():
+        dir_path = pwd / dir_path if pwd else dir_path.absolute()
+
+    if not dir_path.is_dir():
+        raise ValueError(f"Not a directory: {dir_path}")
+    if max_depth < -1:
+        raise ValueError(f"max_depth must be >= -1, got {max_depth}")
+
+    # Get all files recursively
+    all_files = get_dir_entries(
+        dir_path,
+        pwd=pwd,
+        recursive=True,
+        include_files=True,
+        include_dirs=False,
+        filter_func=filter_func,
+    )
+
+    hashes: dict[str, str] = {}
+    hashable_summary = []
+
+    # Calculate hashes for all files
+    for subfile in all_files:
+        subfile_path = dir_path / subfile
+        sha256_hash = hash_file(subfile_path)
+        hashes[subfile] = sha256_hash
+        hashable_summary.append(f"{sha256_hash}  ./{subfile}")
+
+    # Calculate hashes for all directories
+    subdirs = get_dir_entries(
+        dir_path,
+        pwd=pwd,
+        recursive=True,
+        include_files=False,
+        include_dirs=True,
+        include_hidden=False,
+        filter_func=filter_func,
+        max_depth=max_depth,
+    )
+
+    for subdir in subdirs:
+        subdir_path = dir_path / subdir
+        subdir_hashes = get_dir_hashes(
+            subdir_path,
+            filter_func=filter_func,
+            max_depth=0,
+        )
+        hashes[subdir] = subdir_hashes["."]
+
+    # Filter results by max_depth
+    if max_depth >= 0:
+        hashes = {path: value for path, value in hashes.items() if len(Path(path).parts) <= max_depth + 1}
+
+    # Calculate root directory hash
+    hashable_summary.sort()
+    root_sha256 = hashlib.sha256("\n".join(hashable_summary).encode()).hexdigest()
+    hashes["."] = root_sha256
+
+    return hashes
+
+
+@lru_cache(maxsize=128)
+def get_dir_entries(
+    dir_path: Path,
+    pwd: Path | None = None,
+    recursive: bool = True,
+    include_files: bool = True,
+    include_dirs: bool = True,
+    include_hidden: bool = False,
+    filter_func: Callable | None = None,
+    max_depth: int = -1,
+) -> tuple[str, ...]:
+    """Get filtered list of directory entries."""
+    pwd = Path(pwd) if pwd else None
+    dir_path = Path(dir_path)
+    if not dir_path.is_absolute():
+        dir_path = pwd / dir_path if pwd else dir_path.absolute()
+
+    results = []
+
+    def process_path(path: Path, depth: int):
+        if not include_hidden and path.name.startswith("."):
+            return False
+        if max_depth >= 0 and depth > max_depth:
+            return False
+        if filter_func:
+            info = {
+                "abspath": str(path.absolute()),
+                "relpath": str(path.relative_to(dir_path)),
+            }
+            if not filter_func(info):
+                return False
+        return True
+
+    for path in dir_path.rglob("*") if recursive else dir_path.glob("*"):
+        current_depth = len(path.relative_to(dir_path).parts)
+
+        if path.is_file() and include_files and process_path(path, current_depth):
+            results.append(str(path.relative_to(dir_path)))
+        elif path.is_dir() and include_dirs and process_path(path, current_depth):
+            results.append(str(path.relative_to(dir_path)))
+
+        if not recursive:
+            break
+
+    return tuple(sorted(results))  # Make immutable for caching
+
+
+@lru_cache(maxsize=1024)
+def get_dir_sizes(dir_path: Path, pwd: Path | None = None, **kwargs) -> dict[str, int]:
+    """Calculate sizes for all files and directories recursively."""
+    sizes: dict[str, int] = {}
+    hashes = get_dir_hashes(dir_path, pwd=pwd, **kwargs)
+    dir_path = Path(dir_path)
+
+    for path_key in hashes:
+        full_path = dir_path / path_key
+        if full_path.is_file():
+            sizes[path_key] = full_path.stat().st_size
+        else:
+            total = 0
+            for file_path in full_path.rglob("*"):
+                if file_path.is_file() and not file_path.name.startswith("."):
+                    total += file_path.stat().st_size
+            sizes[path_key + "/"] = total
+
+    return sizes
+
+
+@lru_cache(maxsize=10)
+def get_dir_info(dir_path: Path, pwd: Path | None = None, filter_func: Callable | None = None, max_depth: int = -1) -> dict:
+    """Get detailed information about directory contents including hashes and sizes."""
+    pwd = Path(pwd) if pwd else None
+    dir_path = Path(dir_path)
+    if not dir_path.is_absolute():
+        dir_path = pwd / dir_path if pwd else dir_path.absolute()
+
+    hashes = get_dir_hashes(dir_path, pwd=pwd, filter_func=filter_func, max_depth=max_depth)
+    sizes = get_dir_sizes(str(dir_path), pwd=pwd, filter_func=filter_func, max_depth=max_depth)
+
+    num_total_subpaths = sum(1 for name in hashes if name != ".")
+    details = {}
+
+    for filename, sha256_hash in sorted(hashes.items()):
+        abs_path = (dir_path / filename).resolve()
+        stat_info = abs_path.stat()
+        num_subpaths = sum(1 for p in hashes if p.startswith(filename + "/"))
+        is_dir = abs_path.is_dir()
+        if is_dir:
+            mime_type = "inode/directory"
+            basename = abs_path.name
+            extension = ""
+            num_bytes = sizes[filename + "/"]
+            if filename == ".":
+                num_subpaths = num_total_subpaths
+            else:
+                filename += "/"
+                num_subpaths = num_subpaths
+        else:  # is_file
+            num_subpaths = None
+            mime_type = mimetypes.guess_type(str(abs_path))[0]
+            extension = abs_path.suffix
+            basename = abs_path.name.rsplit(extension, 1)[0]
+            num_bytes = sizes[filename]
+
+        details[filename] = {
+            "basename": basename,
+            "mime_type": mime_type,
+            "extension": extension,
+            "num_subpaths": num_subpaths,
+            "num_bytes": num_bytes,
+            "hash_sha256": sha256_hash,
+            "created_at": datetime.fromtimestamp(stat_info.st_ctime).isoformat(),
+            "modified_at": datetime.fromtimestamp(stat_info.st_mtime).isoformat(),
+        }
+
+        if filter_func and not filter_func(details[filename]):
+            del details[filename]
+
+    return details
+
+
+if __name__ == "__main__":
+    import json
+
+    dir_info = get_dir_info(Path("."), max_depth=6)
+    with open(".hashes.json", "w") as f:
+        json.dump(dir_info, f, indent=4)
+    print("Wrote .hashes.json")
+
+# Example output:
+# {
+#     ".": {
+#         "basename": "misc",
+#         "mime_type": "inode/directory",
+#         "extension": "",
+#         "num_subpaths": 25,
+#         "num_bytes": 214677,
+#         "hash_sha256": "addfacf88b2ff6b564846415fb7b21dcb7e63ee4e911bc0aec255ee354958530",
+#         "created_at": "2024-12-04T00:08:38.537449",
+#         "modified_at": "2024-12-04T00:08:38.537449"
+#     },
+#     "__init__.py": {
+#         "basename": "__init__",
+#         "mime_type": "text/x-python",
+#         "extension": ".py",
+#         "num_subpaths": null,
+#         "num_bytes": 32,
+#         "hash_sha256": "b0e5e7ff17db3b60535cf664282787767c336e3e203a43e21b6326c6fe457551",
+#         "created_at": "2024-10-08T00:51:41.001359",
+#         "modified_at": "2024-10-08T00:51:41.001359"
+#     },
+#     ...
+# }

archivebox/misc/jsonl.py

@@ -32,6 +32,7 @@ import sys
 import json
 from typing import Any, TextIO
 from collections.abc import Iterable, Iterator
+from pathlib import Path
 
 
 # Type constants for JSONL records
@@ -119,6 +120,19 @@ def read_stdin(stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
             yield record
 
 
+def read_file(path: Path) -> Iterator[dict[str, Any]]:
+    """
+    Read JSONL or plain URLs from a file.
+
+    Yields parsed records as dicts.
+    """
+    with open(path) as f:
+        for line in f:
+            record = parse_line(line)
+            if record:
+                yield record
+
+
 def read_args_or_stdin(args: Iterable[str], stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
     """
     Read from CLI arguments if provided, otherwise from stdin.

archivebox/misc/logging_util.py

@@ -5,21 +5,133 @@ __package__ = "archivebox"
 # references Django ORM types. For pre-bootstrap logging primitives use
 # misc/logging.py, which has no archivebox or Django dependencies.
 
+import re
 import os
 import sys
 import time
 
+from math import log
+from multiprocessing import Process
 from pathlib import Path
 
-from typing import Any, TYPE_CHECKING, cast
+from datetime import datetime, timezone
+from typing import Any, Optional, TYPE_CHECKING, cast
 
 if TYPE_CHECKING:
-    pass
+    from archivebox.core.models import Snapshot
 
 from rich import print
 
 from archivebox.config import CONSTANTS
+from archivebox.config.common import get_config
 from archivebox.misc.util import enforce_types
+from archivebox.misc.logging import ANSI
+
+
+class TimedProgress:
+    """Show a progress bar and measure elapsed time until .end() is called"""
+
+    def __init__(self, seconds, prefix="", config=None, **config_kwargs):
+
+        config = config or get_config(**config_kwargs)
+        self.SHOW_PROGRESS = config.SHOW_PROGRESS
+        self.ANSI = config.ANSI
+        self.TERM_WIDTH = config.TERM_WIDTH
+
+        if self.SHOW_PROGRESS:
+            self.p = Process(target=progress_bar, args=(seconds, prefix, self.ANSI))
+            self.p.start()
+
+        self.stats = {"start_ts": datetime.now(timezone.utc), "end_ts": None}
+
+    def end(self):
+        """immediately end progress, clear the progressbar line, and save end_ts"""
+
+        end_ts = datetime.now(timezone.utc)
+        self.stats["end_ts"] = end_ts
+
+        if self.SHOW_PROGRESS:
+            # terminate if we havent already terminated
+            try:
+                # kill the progress bar subprocess
+                try:
+                    self.p.close()  # must be closed *before* its terminnated
+                except (KeyboardInterrupt, SystemExit):
+                    print()
+                    raise
+                except BaseException:  # lgtm [py/catch-base-exception]
+                    pass
+                self.p.terminate()
+                time.sleep(0.1)
+                # sometimes the timer doesn't terminate properly, then blocks at the join until
+                # the full time has elapsed. sending a kill tries to avoid that.
+                try:
+                    self.p.kill()
+                except Exception:
+                    pass
+
+                # clear whole terminal line
+                try:
+                    sys.stdout.write("\r{}{}\r".format((" " * self.TERM_WIDTH), self.ANSI["reset"]))
+                except (OSError, BrokenPipeError):
+                    # ignore when the parent proc has stopped listening to our stdout
+                    pass
+            except ValueError:
+                pass
+
+
+@enforce_types
+def progress_bar(seconds: int, prefix: str = "", ANSI: dict[str, str] = ANSI, config=None, **config_kwargs) -> None:
+    """show timer in the form of progress bar, with percentage and seconds remaining"""
+    output_buf = sys.stdout or sys.__stdout__ or sys.stderr or sys.__stderr__
+    chunk = "█" if output_buf and output_buf.encoding.upper() == "UTF-8" else "#"
+    config = config or get_config(**config_kwargs)
+    last_width = config.TERM_WIDTH
+    chunks = last_width - len(prefix) - 20  # number of progress chunks to show (aka max bar width)
+    try:
+        for s in range(seconds * chunks):
+            max_width = config.TERM_WIDTH
+            if max_width < last_width:
+                # when the terminal size is shrunk, we have to write a newline
+                # otherwise the progress bar will keep wrapping incorrectly
+                sys.stdout.write("\r\n")
+                sys.stdout.flush()
+            chunks = max_width - len(prefix) - 20
+            pct_complete = s / chunks / seconds * 100
+            log_pct = (log(pct_complete or 1, 10) / 2) * 100  # everyone likes faster progress bars ;)
+            bar_width = round(log_pct / (100 / chunks))
+            last_width = max_width
+
+            # ████████████████████           0.9% (1/60sec)
+            sys.stdout.write(
+                "\r{}{}{}{} {}% ({}/{}sec)".format(
+                    prefix,
+                    ANSI["green" if pct_complete < 80 else "lightyellow"],
+                    (chunk * bar_width).ljust(chunks),
+                    ANSI["reset"],
+                    round(pct_complete, 1),
+                    round(s / chunks),
+                    seconds,
+                ),
+            )
+            sys.stdout.flush()
+            time.sleep(1 / chunks)
+
+        # ██████████████████████████████████ 100.0% (60/60sec)
+        sys.stdout.write(
+            "\r{}{}{}{} {}% ({}/{}sec)".format(
+                prefix,
+                ANSI["red"],
+                chunk * chunks,
+                ANSI["reset"],
+                100.0,
+                seconds,
+                seconds,
+            ),
+        )
+        sys.stdout.flush()
+    except (KeyboardInterrupt, BrokenPipeError):
+        print()
 
 
 def log_list_started(filter_patterns: list[str] | None, filter_type: str):
@@ -124,6 +236,152 @@ def format_duration(seconds: float) -> str:
         return f"{hours}hr {minutes}min" if minutes else f"{hours}hr"
 
 
+@enforce_types
+def truncate_url(url: str, max_length: int = 60) -> str:
+    """Truncate URL to max_length, keeping domain and adding ellipsis."""
+    if len(url) <= max_length:
+        return url
+    # Try to keep the domain and beginning of path
+    if "://" in url:
+        protocol, rest = url.split("://", 1)
+        if "/" in rest:
+            domain, path = rest.split("/", 1)
+            available = max_length - len(protocol) - len(domain) - 6  # for "://", "/", "..."
+            if available > 10:
+                return f"{protocol}://{domain}/{path[:available]}..."
+    # Fallback: just truncate
+    return url[: max_length - 3] + "..."
+
+
+@enforce_types
+def log_worker_event(
+    worker_type: str,
+    event: str,
+    indent_level: int = 0,
+    pid: int | None = None,
+    worker_id: str | None = None,
+    url: str | None = None,
+    plugin: str | None = None,
+    metadata: dict[str, Any] | None = None,
+    error: Exception | None = None,
+) -> None:
+    """
+    Log a worker event with structured metadata and indentation.
+
+    Args:
+        worker_type: Type of worker (Orchestrator, CrawlWorker, SnapshotWorker)
+        event: Event name (Starting, Completed, Failed, etc.)
+        indent_level: Indentation level (0=Orchestrator, 1=CrawlWorker, 2=SnapshotWorker)
+        pid: Process ID
+        worker_id: Worker ID (UUID for workers)
+        url: URL being processed (for SnapshotWorker)
+        plugin: Plugin name (for hook processes)
+        metadata: Dict of metadata to show in curly braces
+        error: Exception if event is an error
+    """
+    indent = "    " * indent_level
+
+    from rich.markup import escape
+
+    # Build worker identifier (without URL/plugin)
+    worker_parts = [worker_type]
+    # Don't add pid/worker_id for DB operations (they happen in whatever process is running)
+    if pid and worker_type != "DB":
+        worker_parts.append(f"pid={pid}")
+    if worker_id and worker_type in ("CrawlWorker", "Orchestrator") and worker_type != "DB":
+        worker_parts.append(f"id={worker_id}")
+
+    # Build worker label parts for brackets (shown inside brackets)
+    worker_label_base = worker_parts[0]
+    worker_bracket_content = ", ".join(worker_parts[1:]) if len(worker_parts) > 1 else None
+
+    # Build URL/plugin display (shown AFTER the label, outside brackets)
+    url_extractor_parts = []
+    if url:
+        url_extractor_parts.append(f"url: {escape(url)}")
+    if plugin:
+        url_extractor_parts.append(f"extractor: {escape(plugin)}")
+
+    url_extractor_str = " | ".join(url_extractor_parts) if url_extractor_parts else ""
+
+    # Build metadata string
+    metadata_str = ""
+    if metadata:
+        # Format metadata nicely
+        meta_parts = []
+        for k, v in metadata.items():
+            if isinstance(v, float):
+                # Format floats nicely (durations, sizes)
+                if "duration" in k.lower():
+                    meta_parts.append(f"{k}: {format_duration(v)}")
+                elif "size" in k.lower():
+                    meta_parts.append(f"{k}: {printable_filesize(int(v))}")
+                else:
+                    meta_parts.append(f"{k}: {v:.2f}")
+            elif isinstance(v, int):
+                # Format integers - check if it's a size
+                if "size" in k.lower() or "bytes" in k.lower():
+                    meta_parts.append(f"{k}: {printable_filesize(v)}")
+                else:
+                    meta_parts.append(f"{k}: {v}")
+            elif isinstance(v, (list, tuple)):
+                meta_parts.append(f"{k}: {len(v)}")
+            else:
+                meta_parts.append(f"{k}: {v}")
+        metadata_str = " | ".join(meta_parts)
+
+    # Determine color based on event
+    color = "white"
+    if event in ("Starting...", "Started", "STARTED", "Started in background"):
+        color = "green"
+    elif event.startswith("Created"):
+        color = "cyan"  # DB creation events
+    elif event in ("Completed", "COMPLETED", "All work complete"):
+        color = "blue"
+    elif event in ("Failed", "ERROR", "Failed to spawn worker"):
+        color = "red"
+    elif event in ("Shutting down", "SHUTDOWN"):
+        color = "grey53"
+
+    # Build final message
+    error_str = f" {type(error).__name__}: {error}" if error else ""
+    from archivebox.misc.logging import CONSOLE, STDERR
+    from rich.text import Text
+
+    # Create a Rich Text object for proper formatting
+    # Text.append() treats content as literal (no markup parsing)
+    text = Text()
+    text.append(indent)
+    text.append(worker_label_base, style=color)
+
+    # Add bracketed content if present (using Text.append to avoid markup issues)
+    if worker_bracket_content:
+        text.append("[", style=color)
+        text.append(worker_bracket_content, style=color)
+        text.append("]", style=color)
+
+    text.append(f" {event}{error_str}", style=color)
+
+    # Add URL/plugin info first (more important)
+    if url_extractor_str:
+        text.append(f" | {url_extractor_str}")
+
+    # Then add other metadata
+    if metadata_str:
+        text.append(f" | {metadata_str}")
+
+    # Stdout is reserved for JSONL records whenever commands are piped together.
+    # Route worker/DB progress to stderr in non-TTY contexts so pipelines like
+    # `archivebox snapshot list | archivebox run` keep stdout machine-readable.
+    output_console = CONSOLE if sys.stdout.isatty() else STDERR
+    output_console.print(text, soft_wrap=True)
+
+
+@enforce_types
+def printable_folders(folders: dict[str, Optional["Snapshot"]], with_headers: bool = False) -> str:
+    return "\n".join(f'{folder} {snapshot and snapshot.url} "{snapshot and snapshot.title}"' for folder, snapshot in folders.items())
+
+
 @enforce_types
 def printable_config(config: dict, prefix: str = "") -> str:
     return f"\n{prefix}".join(f"{key}={val}" for key, val in config.items() if not (isinstance(val, dict) or callable(val)))
@@ -174,93 +432,32 @@ def printable_folder_status(name: str, folder: dict) -> str:
     )
 
 
-def _warn_background_cleanup(context: str, err: BaseException) -> None:
-    from archivebox.misc.logging import STDERR
+@enforce_types
+def printable_dependency_version(name: str, dependency: dict) -> str:
+    color, symbol, note, version = "red", "X", "invalid", "?"
 
-    STDERR.print(f"[yellow][!] {context}: {err!s}[/yellow]")
+    if dependency["enabled"]:
+        if dependency["is_valid"]:
+            color, symbol, note = "green", "√", "valid"
 
+            parsed_version_num = re.search(r"[\d\.]+", dependency["version"])
+            if parsed_version_num:
+                version = f"v{parsed_version_num[0]}"
+    else:
+        color, symbol, note, version = "lightyellow", "-", "disabled", "-"
 
-def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep_running=None):
-    """Tail multiple log files simultaneously, interleaving their output.
+    path = pretty_path(dependency["path"])
 
-    Args:
-        log_files: List of log file paths to tail
-        follow: Whether to keep following (True) or just read existing content (False)
-        proc: Optional subprocess.Popen object - stop tailing when this process exits
-    """
-    import re
-    from archivebox.config.common import rprint as print
-
-    # Convert relative paths to absolute paths
-    log_paths = []
-    for log_file in log_files:
-        log_path = Path(log_file)
-        if not log_path.is_absolute():
-            log_path = CONSTANTS.DATA_DIR / log_path
-
-        # Create log file if it doesn't exist
-        if not log_path.exists():
-            log_path.parent.mkdir(parents=True, exist_ok=True)
-            log_path.touch()
-
-        log_paths.append(log_path)
-
-    # Open all log files
-    file_handles = []
-    for log_path in log_paths:
-        try:
-            f = log_path.open()
-            # Seek to end - only show NEW logs from now on, not old logs
-            f.seek(0, 2)  # Go to end
-
-            file_handles.append((log_path, f))
-            print(f"    [tailing {log_path.name}]")
-        except OSError as e:
-            sys.stderr.write(f"Warning: Could not open {log_path}: {e}\n")
-
-    if not file_handles:
-        sys.stderr.write("No log files could be opened\n")
-        return
-
-    print()
-
-    try:
-        while follow:
-            if keep_running is not None and not keep_running():
-                print("\n[newer ArchiveBox process is now running the orchestrator and server]")
-                return "transferred"
-
-            # Check if the monitored process has exited
-            if proc is not None and proc.poll() is not None:
-                print(f"\n[server process exited with code {proc.returncode}]")
-                return "exited"
-
-            had_output = False
-            # Read ALL available lines from all files (not just one per iteration)
-            for log_path, f in file_handles:
-                while True:
-                    line = f.readline()
-                    if not line:
-                        break  # No more lines available in this file
-                    had_output = True
-                    # Strip ANSI codes if present (supervisord does this but just in case)
-                    line_clean = re.sub(r"\x1b\[[0-9;]*m", "", line.rstrip())
-                    if line_clean:
-                        print(line_clean)
-
-            # Small sleep to avoid busy-waiting (only when no output)
-            if not had_output:
-                time.sleep(0.05)
-
-    except (KeyboardInterrupt, BrokenPipeError, OSError):
-        return "interrupted"  # Let the caller handle the cleanup message
-    except SystemExit:
-        return "interrupted"
-    finally:
-        # Close all file handles
-        for _, f in file_handles:
-            try:
-                f.close()
-            except OSError as err:
-                _warn_background_cleanup("Could not close worker log file", err)
-    return "stopped"
+    return " ".join(
+        (
+            ANSI[color],
+            symbol,
+            ANSI["reset"],
+            name.ljust(21),
+            version.ljust(14),
+            ANSI[color],
+            note.ljust(8),
+            ANSI["reset"],
+            path.ljust(76),
+        ),
+    )

archivebox/misc/replay_preview.py

@@ -1,319 +0,0 @@
-"""Transform captured text, Markdown, and images into replay documents.
-
-HTTP caching, range requests, and replay security headers belong to serve_static;
-these helpers only render content and resolve references to captured responses.
-"""
-
-import html
-import posixpath
-import re
-from pathlib import Path
-from urllib.parse import quote, urljoin
-
-from markdown_it import MarkdownIt
-
-IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\s(?:src|data-src)=["\'])([^"\']+)(["\'])', re.IGNORECASE)
-
-
-TRANSFORMED_HTML_PREVIEW_STYLE = """<style id="archivebox-static-html-preview-style">
-html {
-    width: 100%;
-    min-width: 100%;
-    background: #fff;
-}
-body {
-    box-sizing: border-box;
-    width: min(100%, 72rem);
-    max-width: none;
-    min-height: 100vh;
-    margin: 0 auto;
-    padding: clamp(1rem, 3vw, 2rem);
-    background: #fff;
-    color: #111827;
-    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
-    line-height: 1.55;
-}
-body > * {
-    max-width: 100%;
-}
-img:not([width]):not([height]) {
-    max-width: min(100%, 12rem);
-    max-height: 12rem;
-    width: auto;
-    height: auto;
-    object-fit: contain;
-}
-a > img:not([width]):not([height]) {
-    max-width: min(100%, 2.5rem);
-    max-height: 2.5rem;
-}
-</style>"""
-
-
-MARKDOWN_INLINE_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)\s]+(?:\([^)]*\)[^)\s]*)*)\)")
-
-
-HTML_BODY_RE = re.compile(r"<body[^>]*>(.*)</body>", flags=re.IGNORECASE | re.DOTALL)
-
-
-def _extract_markdown_candidate(text: str) -> str:
-    candidate = text
-    body_match = HTML_BODY_RE.search(candidate)
-    if body_match:
-        candidate = body_match.group(1)
-    candidate = re.sub(r"^\s*<p[^>]*>", "", candidate, flags=re.IGNORECASE)
-    candidate = re.sub(r"</p>\s*$", "", candidate, flags=re.IGNORECASE)
-    return candidate.strip()
-
-
-def _looks_like_markdown(text: str) -> bool:
-    lower = text.lower()
-    if "<html" in lower and "<head" in lower and "</body>" in lower:
-        return False
-    md_markers = 0
-    md_markers += len(re.findall(r"^\s{0,3}#{1,6}\s+\S", text, flags=re.MULTILINE))
-    md_markers += len(re.findall(r"^\s*[-*+]\s+\S", text, flags=re.MULTILINE))
-    md_markers += len(re.findall(r"^\s*\d+\.\s+\S", text, flags=re.MULTILINE))
-    md_markers += text.count("[TOC]")
-    md_markers += len(MARKDOWN_INLINE_LINK_RE.findall(text))
-    md_markers += text.count("\n---") + text.count("\n***")
-    return md_markers >= 6
-
-
-def _render_text_preview_document(text: str, title: str) -> str:
-    escaped_title = html.escape(title)
-    escaped_text = html.escape(text)
-    return f"""<!doctype html>
-<html lang="en">
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>{escaped_title}</title>
-    <style>
-        :root {{
-            color-scheme: dark;
-        }}
-        html, body {{
-            margin: 0;
-            padding: 0;
-            background: #111;
-            color: #f3f3f3;
-            font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace;
-        }}
-        .archivebox-text-preview-header {{
-            position: sticky;
-            top: 0;
-            z-index: 1;
-            padding: 10px 14px;
-            font-size: 12px;
-            line-height: 1.4;
-            color: #bbb;
-            background: rgba(17, 17, 17, 0.96);
-            border-bottom: 1px solid rgba(255, 255, 255, 0.08);
-            backdrop-filter: blur(8px);
-        }}
-        .archivebox-text-preview {{
-            margin: 0;
-            padding: 14px;
-            white-space: pre-wrap;
-            word-break: break-word;
-            tab-size: 2;
-            line-height: 1.45;
-            font-size: 13px;
-        }}
-    </style>
-</head>
-<body>
-    <div class="archivebox-text-preview-header">{escaped_title}</div>
-    <pre class="archivebox-text-preview">{escaped_text}</pre>
-</body>
-</html>"""
-
-
-def _render_image_preview_document(image_url: str, title: str) -> str:
-    escaped_title = html.escape(title)
-    escaped_url = html.escape(image_url, quote=True)
-    return f"""<!doctype html>
-<html lang="en">
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>{escaped_title}</title>
-    <style>
-        :root {{
-            color-scheme: dark;
-        }}
-        html, body {{
-            margin: 0;
-            padding: 0;
-            width: 100%;
-            min-height: 100%;
-            background: #fff;
-        }}
-        body {{
-            overflow: auto;
-        }}
-        .archivebox-image-preview {{
-            width: 100%;
-            min-width: 100%;
-            min-height: 100vh;
-            display: flex;
-            flex-direction: column;
-            align-items: center;
-            justify-content: flex-start;
-            box-sizing: border-box;
-        }}
-        .archivebox-image-preview img {{
-            display: block;
-            width: auto;
-            max-width: 100%;
-            height: auto;
-            margin: 0 auto;
-        }}
-    </style>
-</head>
-<body>
-    <div class="archivebox-image-preview">
-        <img src="{escaped_url}" alt="{escaped_title}">
-    </div>
-</body>
-</html>"""
-
-
-def _encoded_responses_image_url(image_url: str, page_url: str | None) -> str | None:
-    raw_url = str(image_url or "").strip()
-    if not raw_url or raw_url.startswith(("#", "data:", "blob:", "about:", "javascript:")):
-        return None
-
-    absolute_url = urljoin(page_url or "", raw_url)
-    if not absolute_url.startswith(("http://", "https://")):
-        return None
-
-    return quote(absolute_url, safe="").replace("%", "_")
-
-
-def _index_responses_paths_for_html_images(
-    snapshot_root: Path,
-    html_rel_path: str,
-    encoded_urls: set[str],
-) -> dict[str, str]:
-    responses_root = snapshot_root / "responses"
-    if not encoded_urls or not responses_root.is_dir():
-        return {}
-
-    best_matches: dict[str, str] = {}
-    image_matches: set[str] = set()
-    image_suffixes = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"}
-    for candidate in responses_root.rglob("*"):
-        if not candidate.is_file():
-            continue
-        try:
-            rel_path = candidate.relative_to(snapshot_root)
-        except ValueError:
-            continue
-        relative_match = posixpath.relpath(rel_path.as_posix(), start=posixpath.dirname(html_rel_path) or ".")
-        candidate_name = candidate.name
-        is_image = candidate.suffix.lower() in image_suffixes
-        for encoded_url in encoded_urls:
-            if encoded_url in image_matches or f"__GET__{encoded_url}" not in candidate_name:
-                continue
-            best_matches[encoded_url] = relative_match
-            if is_image:
-                # Preserve the old rglob behavior: the first image match wins,
-                # otherwise the last matching response file is used.
-                image_matches.add(encoded_url)
-    return best_matches
-
-
-def _rewrite_html_image_sources_to_responses(
-    html_text: str,
-    snapshot_root: Path,
-    html_rel_path: str,
-    page_url: str | None,
-) -> tuple[str, int]:
-    encoded_urls_by_src: dict[str, str | None] = {}
-    for match in IMG_SRC_ATTR_RE.finditer(html_text):
-        image_url = html.unescape(match.group(2))
-        if image_url not in encoded_urls_by_src:
-            encoded_urls_by_src[image_url] = _encoded_responses_image_url(image_url, page_url)
-
-    response_paths = _index_responses_paths_for_html_images(
-        snapshot_root,
-        html_rel_path,
-        {encoded_url for encoded_url in encoded_urls_by_src.values() if encoded_url},
-    )
-    rewrites = 0
-
-    def replace_src(match: re.Match[str]) -> str:
-        nonlocal rewrites
-        image_url = html.unescape(match.group(2))
-        encoded_url = encoded_urls_by_src.get(image_url)
-        local_path = response_paths.get(encoded_url or "")
-        if not local_path:
-            return match.group(0)
-        rewrites += 1
-        return f"{match.group(1)}{html.escape(local_path, quote=True)}{match.group(3)}"
-
-    return IMG_SRC_ATTR_RE.sub(replace_src, html_text), rewrites
-
-
-def _rewrite_html_image_sources_for_request(
-    request,
-    html_text: str,
-    document_root: Path | None,
-    html_rel_path: str,
-) -> tuple[str, int]:
-    if not document_root:
-        return html_text, 0
-    return _rewrite_html_image_sources_to_responses(
-        html_text,
-        document_root,
-        html_rel_path,
-        request.__dict__.get("archivebox_snapshot_url"),
-    )
-
-
-def _apply_transformed_html_preview_style(html_text: str) -> str:
-    if "archivebox-static-html-preview-style" in html_text:
-        return html_text
-    if re.search(r"</head\s*>", html_text, flags=re.IGNORECASE):
-        return re.sub(r"</head\s*>", f"{TRANSFORMED_HTML_PREVIEW_STYLE}\\g<0>", html_text, count=1, flags=re.IGNORECASE)
-    return f"{TRANSFORMED_HTML_PREVIEW_STYLE}\n{html_text}"
-
-
-def _render_markdown(markdown_text: str) -> str:
-    """Render archived Markdown with stable heading links and optional [TOC]."""
-    parser = MarkdownIt("commonmark").enable("table")
-    tokens = parser.parse(markdown_text)
-    headings = []
-    for index, token in enumerate(tokens):
-        if token.type == "heading_open":
-            title = tokens[index + 1].content
-            slug = re.sub(r"[^A-Za-z0-9]+", "-", title).strip("-") or "section"
-            token.attrSet("id", slug)
-            headings.append(f'<li class="toc-level-{token.tag[1:]}"><a href="#{slug}">{html.escape(title)}</a></li>')
-    toc = '<nav class="toc"><ul>' + "".join(headings) + "</ul></nav>"
-    # Replace only a standalone TOC paragraph, leaving code and inline mentions intact.
-    for index in range(len(tokens) - 2):
-        opening, content, closing = tokens[index : index + 3]
-        if opening.type == "paragraph_open" and content.content == "[TOC]" and closing.type == "paragraph_close":
-            opening.hidden = closing.hidden = True
-            content.type = "html_block"
-            content.content = toc
-    return parser.renderer.render(tokens, parser.options, {})
-
-
-def _render_markdown_document(markdown_text: str) -> str:
-    body = _render_markdown(markdown_text)
-    wrapped = (
-        '<!doctype html><html><head><meta charset="utf-8">'
-        '<meta name="viewport" content="width=device-width,initial-scale=1">'
-        "<style>body{max-width:900px;margin:24px auto;padding:0 16px;"
-        "font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif;"
-        "line-height:1.55;} img{max-width:100%;} pre{background:#f6f6f6;padding:12px;overflow:auto;}"
-        ".toc ul{list-style:none;padding-left:0;} .toc li{margin:4px 0;}</style>"
-        "</head><body>"
-        f"{body}"
-        "</body></html>"
-    )
-    return wrapped

archivebox/misc/serve_static.py

@@ -1,18 +1,21 @@
 import asyncio
 import html
-import io
+import importlib
 import json
 import mimetypes
 import os
 import posixpath
+import queue
 import re
 import stat
 import sys
+import threading
+import time
 import zipfile
+from collections.abc import Callable
 from datetime import UTC, datetime
 from pathlib import Path
-from functools import partial
-from urllib.parse import urlencode
+from urllib.parse import quote, urlencode, urljoin
 
 from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay
 from django.contrib.staticfiles import finders
@@ -25,10 +28,43 @@ from django.utils.translation import gettext as _
 from django.views import static
 
 from archivebox.config.common import get_config
-from archivebox.misc import replay_preview
 from archivebox.misc.logging_util import printable_filesize
 
 _HASHES_CACHE: dict[Path, tuple[float, dict[str, str]]] = {}
+IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\s(?:src|data-src)=["\'])([^"\']+)(["\'])', re.IGNORECASE)
+TRANSFORMED_HTML_PREVIEW_STYLE = """<style id="archivebox-static-html-preview-style">
+html {
+    width: 100%;
+    min-width: 100%;
+    background: #fff;
+}
+body {
+    box-sizing: border-box;
+    width: min(100%, 72rem);
+    max-width: none;
+    min-height: 100vh;
+    margin: 0 auto;
+    padding: clamp(1rem, 3vw, 2rem);
+    background: #fff;
+    color: #111827;
+    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+    line-height: 1.55;
+}
+body > * {
+    max-width: 100%;
+}
+img:not([width]):not([height]) {
+    max-width: min(100%, 12rem);
+    max-height: 12rem;
+    width: auto;
+    height: auto;
+    object-fit: contain;
+}
+a > img:not([width]):not([height]) {
+    max-width: min(100%, 2.5rem);
+    max-height: 2.5rem;
+}
+</style>"""
 
 
 def _load_hash_map(snapshot_dir: Path) -> dict[str, str] | None:
@@ -106,25 +142,33 @@ def _safe_zip_stem(name: str) -> str:
     return safe_name or "archivebox"
 
 
-class _ZipBuffer(io.RawIOBase):
-    """A non-seekable ZIP destination drained after each source block."""
+class _StreamingQueueWriter:
+    """Expose a write-only file-like object so zipfile can stream into a queue."""
 
-    def __init__(self):
-        self.pending = bytearray()
+    def __init__(self, output_queue: queue.Queue[bytes | BaseException | object]) -> None:
+        self.output_queue = output_queue
         self.position = 0
 
-    def write(self, data):
-        self.pending.extend(data)
-        self.position += len(data)
+    def write(self, data: bytes) -> int:
+        if data:
+            self.output_queue.put(data)
+            self.position += len(data)
         return len(data)
 
-    def tell(self):
+    def tell(self) -> int:
         return self.position
 
-    def drain(self):
-        if self.pending:
-            yield bytes(self.pending)
-            self.pending.clear()
+    def flush(self) -> None:
+        return None
+
+    def close(self) -> None:
+        return None
+
+    def writable(self) -> bool:
+        return True
+
+    def seekable(self) -> bool:
+        return False
 
 
 def _iter_visible_files(root: Path):
@@ -136,42 +180,6 @@ def _iter_visible_files(root: Path):
             yield Path(current_root) / filename
 
 
-def _iter_directory_zip(fullpath: Path, root_name: str):
-    # Reading a block and yielding its compressed bytes in the same iterator
-    # provides backpressure and closes all files when a download disconnects.
-    with _ZipBuffer() as buffer:
-        with zipfile.ZipFile(buffer, mode="w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
-            for entry in _iter_visible_files(fullpath):
-                info = zipfile.ZipInfo.from_file(entry, (Path(root_name) / entry.relative_to(fullpath)).as_posix())
-                info.compress_type = archive.compression
-                info.compress_level = archive.compresslevel
-                with entry.open("rb") as source, archive.open(info, "w") as destination:
-                    while chunk := source.read(64 * 1024):
-                        destination.write(chunk)
-                        yield from buffer.drain()
-                yield from buffer.drain()
-        yield from buffer.drain()
-
-
-async def _stream_async(stream):
-    # Django ASGI buffers synchronous iterators. Advance off the event loop,
-    # waiting for an in-flight read before closing its iterator on cancellation.
-    iterator = iter(stream)
-    try:
-        while True:
-            read = asyncio.create_task(asyncio.to_thread(next, iterator, None))
-            try:
-                chunk = await asyncio.shield(read)
-            except asyncio.CancelledError:
-                await read
-                raise
-            if chunk is None:
-                break
-            yield chunk
-    finally:
-        iterator.close()
-
-
 def _build_directory_zip_response(
     fullpath: Path,
     path: str,
@@ -181,11 +189,74 @@ def _build_directory_zip_response(
     config=None,
 ) -> StreamingHttpResponse:
     root_name = _safe_zip_stem(fullpath.name or Path(path).name or "archivebox")
-    stream = _iter_directory_zip(fullpath, root_name)
+    sentinel = object()
+    output_queue: queue.Queue[bytes | BaseException | object] = queue.Queue(maxsize=8)
+    initial_chunk_target = 64 * 1024
+    initial_chunk_wait = 0.05
 
-    response = StreamingHttpResponse(stream, content_type="application/zip")
-    if use_async_stream:
-        response.streaming_content = _stream_async(stream)
+    def build_zip() -> None:
+        # zipfile wants a write-only file object. Feed those bytes straight into
+        # a queue so the response can stream them out as soon as they are ready.
+        writer = _StreamingQueueWriter(output_queue)
+        try:
+            with zipfile.ZipFile(writer, mode="w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as zip_file:
+                for entry in _iter_visible_files(fullpath):
+                    rel_parts = entry.relative_to(fullpath).parts
+                    arcname = Path(root_name, *rel_parts).as_posix()
+                    zip_file.write(entry, arcname)
+        except (OSError, RuntimeError, TypeError, ValueError, zipfile.BadZipFile) as err:
+            output_queue.put(err)
+        finally:
+            output_queue.put(sentinel)
+
+    threading.Thread(target=build_zip, name=f"zip-stream-{root_name}", daemon=True).start()
+
+    def iter_zip_chunks():
+        # Emit a meaningful first chunk quickly so browsers show the download
+        # immediately instead of waiting on dozens of tiny ZIP header writes.
+        first_chunk = bytearray()
+        initial_deadline = time.monotonic() + initial_chunk_wait
+
+        while True:
+            timeout = max(initial_deadline - time.monotonic(), 0) if len(first_chunk) < initial_chunk_target else None
+            try:
+                chunk = output_queue.get(timeout=timeout) if timeout is not None else output_queue.get()
+            except queue.Empty:
+                if first_chunk:
+                    yield bytes(first_chunk)
+                    first_chunk.clear()
+                    continue
+                chunk = output_queue.get()
+
+            if chunk is sentinel:
+                if first_chunk:
+                    yield bytes(first_chunk)
+                break
+            if isinstance(chunk, BaseException):
+                raise chunk
+            if len(first_chunk) < initial_chunk_target:
+                first_chunk.extend(chunk)
+                if len(first_chunk) >= initial_chunk_target or time.monotonic() >= initial_deadline:
+                    yield bytes(first_chunk)
+                    first_chunk.clear()
+                continue
+            yield chunk
+
+    async def stream_zip_async():
+        # Django ASGI buffers sync StreamingHttpResponse iterators by consuming
+        # them into a list. Drive the same sync iterator from a worker thread so
+        # Daphne can send each chunk as it arrives instead of buffering the ZIP.
+        iterator = iter(iter_zip_chunks())
+        while True:
+            chunk = await asyncio.to_thread(next, iterator, None)
+            if chunk is None:
+                break
+            yield chunk
+
+    response = StreamingHttpResponse(
+        stream_zip_async() if use_async_stream else iter_zip_chunks(),
+        content_type="application/zip",
+    )
     response.headers["Content-Disposition"] = f'attachment; filename="{root_name}.zip"'
     response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
     response.headers["Last-Modified"] = http_date(fullpath.stat().st_mtime)
@@ -199,6 +270,18 @@ def _build_directory_zip_response(
     )
 
 
+async def _stream_ranged_file_async(ranged_file: "RangedFileReader"):
+    iterator = iter(ranged_file)
+    try:
+        while True:
+            chunk = await asyncio.to_thread(next, iterator, None)
+            if chunk is None:
+                break
+            yield chunk
+    finally:
+        ranged_file.close()
+
+
 def _render_directory_index(request, path: str, fullpath: Path) -> HttpResponse:
     try:
         template = loader.select_template(
@@ -273,6 +356,17 @@ mimetypes.add_type("image/svg+xml", ".svg")
 mimetypes.add_type("multipart/related", ".mhtml")
 mimetypes.add_type("multipart/related", ".mht")
 
+try:
+    _markdown = importlib.import_module("markdown").markdown
+except ImportError:
+    _markdown: Callable[..., str] | None = None
+
+MARKDOWN_INLINE_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)\s]+(?:\([^)]*\)[^)\s]*)*)\)")
+MARKDOWN_INLINE_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
+MARKDOWN_BOLD_RE = re.compile(r"\*\*([^*]+)\*\*")
+MARKDOWN_ITALIC_RE = re.compile(r"(?<!\*)\*([^*]+)\*(?!\*)")
+HTML_TAG_RE = re.compile(r"<[A-Za-z][^>]*>")
+HTML_BODY_RE = re.compile(r"<body[^>]*>(.*)</body>", flags=re.IGNORECASE | re.DOTALL)
 RISKY_REPLAY_MIMETYPES = {
     "text/html",
     "application/xhtml+xml",
@@ -286,6 +380,231 @@ RISKY_REPLAY_MARKERS = (
 )
 
 
+def _extract_markdown_candidate(text: str) -> str:
+    candidate = text
+    body_match = HTML_BODY_RE.search(candidate)
+    if body_match:
+        candidate = body_match.group(1)
+    candidate = re.sub(r"^\s*<p[^>]*>", "", candidate, flags=re.IGNORECASE)
+    candidate = re.sub(r"</p>\s*$", "", candidate, flags=re.IGNORECASE)
+    return candidate.strip()
+
+
+def _looks_like_markdown(text: str) -> bool:
+    lower = text.lower()
+    if "<html" in lower and "<head" in lower and "</body>" in lower:
+        return False
+    md_markers = 0
+    md_markers += len(re.findall(r"^\s{0,3}#{1,6}\s+\S", text, flags=re.MULTILINE))
+    md_markers += len(re.findall(r"^\s*[-*+]\s+\S", text, flags=re.MULTILINE))
+    md_markers += len(re.findall(r"^\s*\d+\.\s+\S", text, flags=re.MULTILINE))
+    md_markers += text.count("[TOC]")
+    md_markers += len(MARKDOWN_INLINE_LINK_RE.findall(text))
+    md_markers += text.count("\n---") + text.count("\n***")
+    return md_markers >= 6
+
+
+def _render_text_preview_document(text: str, title: str) -> str:
+    escaped_title = html.escape(title)
+    escaped_text = html.escape(text)
+    return f"""<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>{escaped_title}</title>
+    <style>
+        :root {{
+            color-scheme: dark;
+        }}
+        html, body {{
+            margin: 0;
+            padding: 0;
+            background: #111;
+            color: #f3f3f3;
+            font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace;
+        }}
+        .archivebox-text-preview-header {{
+            position: sticky;
+            top: 0;
+            z-index: 1;
+            padding: 10px 14px;
+            font-size: 12px;
+            line-height: 1.4;
+            color: #bbb;
+            background: rgba(17, 17, 17, 0.96);
+            border-bottom: 1px solid rgba(255, 255, 255, 0.08);
+            backdrop-filter: blur(8px);
+        }}
+        .archivebox-text-preview {{
+            margin: 0;
+            padding: 14px;
+            white-space: pre-wrap;
+            word-break: break-word;
+            tab-size: 2;
+            line-height: 1.45;
+            font-size: 13px;
+        }}
+    </style>
+</head>
+<body>
+    <div class="archivebox-text-preview-header">{escaped_title}</div>
+    <pre class="archivebox-text-preview">{escaped_text}</pre>
+</body>
+</html>"""
+
+
+def _render_image_preview_document(image_url: str, title: str) -> str:
+    escaped_title = html.escape(title)
+    escaped_url = html.escape(image_url, quote=True)
+    return f"""<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>{escaped_title}</title>
+    <style>
+        :root {{
+            color-scheme: dark;
+        }}
+        html, body {{
+            margin: 0;
+            padding: 0;
+            width: 100%;
+            min-height: 100%;
+            background: #fff;
+        }}
+        body {{
+            overflow: auto;
+        }}
+        .archivebox-image-preview {{
+            width: 100%;
+            min-width: 100%;
+            min-height: 100vh;
+            display: flex;
+            flex-direction: column;
+            align-items: center;
+            justify-content: flex-start;
+            box-sizing: border-box;
+        }}
+        .archivebox-image-preview img {{
+            display: block;
+            width: auto;
+            max-width: 100%;
+            height: auto;
+            margin: 0 auto;
+        }}
+    </style>
+</head>
+<body>
+    <div class="archivebox-image-preview">
+        <img src="{escaped_url}" alt="{escaped_title}">
+    </div>
+</body>
+</html>"""
+
+
+def _encoded_responses_image_url(image_url: str, page_url: str | None) -> str | None:
+    raw_url = str(image_url or "").strip()
+    if not raw_url or raw_url.startswith(("#", "data:", "blob:", "about:", "javascript:")):
+        return None
+
+    absolute_url = urljoin(page_url or "", raw_url)
+    if not absolute_url.startswith(("http://", "https://")):
+        return None
+
+    return quote(absolute_url, safe="").replace("%", "_")
+
+
+def _index_responses_paths_for_html_images(
+    snapshot_root: Path,
+    html_rel_path: str,
+    encoded_urls: set[str],
+) -> dict[str, str]:
+    responses_root = snapshot_root / "responses"
+    if not encoded_urls or not responses_root.is_dir():
+        return {}
+
+    best_matches: dict[str, str] = {}
+    image_matches: set[str] = set()
+    image_suffixes = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"}
+    for candidate in responses_root.rglob("*"):
+        if not candidate.is_file():
+            continue
+        try:
+            rel_path = candidate.relative_to(snapshot_root)
+        except ValueError:
+            continue
+        relative_match = posixpath.relpath(rel_path.as_posix(), start=posixpath.dirname(html_rel_path) or ".")
+        candidate_name = candidate.name
+        is_image = candidate.suffix.lower() in image_suffixes
+        for encoded_url in encoded_urls:
+            if encoded_url in image_matches or f"__GET__{encoded_url}" not in candidate_name:
+                continue
+            best_matches[encoded_url] = relative_match
+            if is_image:
+                # Preserve the old rglob behavior: the first image match wins,
+                # otherwise the last matching response file is used.
+                image_matches.add(encoded_url)
+    return best_matches
+
+
+def _rewrite_html_image_sources_to_responses(
+    html_text: str,
+    snapshot_root: Path,
+    html_rel_path: str,
+    page_url: str | None,
+) -> tuple[str, int]:
+    encoded_urls_by_src: dict[str, str | None] = {}
+    for match in IMG_SRC_ATTR_RE.finditer(html_text):
+        image_url = html.unescape(match.group(2))
+        if image_url not in encoded_urls_by_src:
+            encoded_urls_by_src[image_url] = _encoded_responses_image_url(image_url, page_url)
+
+    response_paths = _index_responses_paths_for_html_images(
+        snapshot_root,
+        html_rel_path,
+        {encoded_url for encoded_url in encoded_urls_by_src.values() if encoded_url},
+    )
+    rewrites = 0
+
+    def replace_src(match: re.Match[str]) -> str:
+        nonlocal rewrites
+        image_url = html.unescape(match.group(2))
+        encoded_url = encoded_urls_by_src.get(image_url)
+        local_path = response_paths.get(encoded_url or "")
+        if not local_path:
+            return match.group(0)
+        rewrites += 1
+        return f"{match.group(1)}{html.escape(local_path, quote=True)}{match.group(3)}"
+
+    return IMG_SRC_ATTR_RE.sub(replace_src, html_text), rewrites
+
+
+def _rewrite_html_image_sources_for_request(
+    request,
+    html_text: str,
+    document_root: Path | None,
+    html_rel_path: str,
+) -> tuple[str, int]:
+    if not document_root:
+        return html_text, 0
+    return _rewrite_html_image_sources_to_responses(
+        html_text,
+        document_root,
+        html_rel_path,
+        request.__dict__.get("archivebox_snapshot_url"),
+    )
+
+
+def _apply_transformed_html_preview_style(html_text: str) -> str:
+    if "archivebox-static-html-preview-style" in html_text:
+        return html_text
+    if re.search(r"</head\s*>", html_text, flags=re.IGNORECASE):
+        return re.sub(r"</head\s*>", f"{TRANSFORMED_HTML_PREVIEW_STYLE}\\g<0>", html_text, count=1, flags=re.IGNORECASE)
+    return f"{TRANSFORMED_HTML_PREVIEW_STYLE}\n{html_text}"
+
+
 def _set_transformed_response_headers(response, fullpath: Path, statobj: os.stat_result, encoding: str | None, config) -> None:
     response.headers["Last-Modified"] = http_date(statobj.st_mtime)
     response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
@@ -294,6 +613,177 @@ def _set_transformed_response_headers(response, fullpath: Path, statobj: os.stat
         response.headers["Content-Encoding"] = encoding
 
 
+def _render_markdown_fallback(text: str) -> str:
+    if _markdown is not None and not HTML_TAG_RE.search(text):
+        try:
+            return _markdown(
+                text,
+                extensions=["extra", "toc", "sane_lists"],
+                output_format="html",
+            )
+        except (ImportError, RuntimeError, ValueError):
+            pass
+
+    lines = text.splitlines()
+    headings = []
+
+    def slugify(value: str) -> str:
+        slug = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-")
+        return slug or "section"
+
+    for raw_line in lines:
+        heading_match = re.match(r"^\s{0,3}(#{1,6})\s+(.*)$", raw_line)
+        if heading_match:
+            level = len(heading_match.group(1))
+            content = heading_match.group(2).strip()
+            headings.append((level, content, slugify(content)))
+
+    html_lines = []
+    in_code = False
+    in_ul = False
+    in_ol = False
+    in_blockquote = False
+
+    def render_inline(markup: str) -> str:
+        content = MARKDOWN_INLINE_IMAGE_RE.sub(r'<img alt="\1" src="\2">', markup)
+        content = MARKDOWN_INLINE_LINK_RE.sub(r'<a href="\2">\1</a>', content)
+        content = MARKDOWN_BOLD_RE.sub(r"<strong>\1</strong>", content)
+        content = MARKDOWN_ITALIC_RE.sub(r"<em>\1</em>", content)
+        return content
+
+    def close_lists():
+        nonlocal in_ul, in_ol
+        if in_ul:
+            html_lines.append("</ul>")
+            in_ul = False
+        if in_ol:
+            html_lines.append("</ol>")
+            in_ol = False
+
+    for raw_line in lines:
+        line = raw_line.rstrip("\n")
+        stripped = line.strip()
+
+        if stripped.startswith("```"):
+            if in_code:
+                html_lines.append("</code></pre>")
+                in_code = False
+            else:
+                close_lists()
+                if in_blockquote:
+                    html_lines.append("</blockquote>")
+                    in_blockquote = False
+                html_lines.append("<pre><code>")
+                in_code = True
+            continue
+
+        if in_code:
+            html_lines.append(html.escape(line))
+            continue
+
+        if not stripped:
+            close_lists()
+            if in_blockquote:
+                html_lines.append("</blockquote>")
+                in_blockquote = False
+            html_lines.append("<br/>")
+            continue
+
+        heading_match = re.match(r"^\s*((?:<[^>]+>\s*)*)(#{1,6})\s+(.*)$", line)
+        if heading_match:
+            close_lists()
+            if in_blockquote:
+                html_lines.append("</blockquote>")
+                in_blockquote = False
+            leading_tags = heading_match.group(1).strip()
+            level = len(heading_match.group(2))
+            content = heading_match.group(3).strip()
+            if leading_tags:
+                html_lines.append(leading_tags)
+            html_lines.append(f'<h{level} id="{slugify(content)}">{render_inline(content)}</h{level}>')
+            continue
+
+        if stripped in ("---", "***"):
+            close_lists()
+            html_lines.append("<hr/>")
+            continue
+
+        if stripped.startswith("> "):
+            if not in_blockquote:
+                close_lists()
+                html_lines.append("<blockquote>")
+                in_blockquote = True
+            content = stripped[2:]
+            html_lines.append(render_inline(content))
+            continue
+        else:
+            if in_blockquote:
+                html_lines.append("</blockquote>")
+                in_blockquote = False
+
+        ul_match = re.match(r"^\s*[-*+]\s+(.*)$", line)
+        if ul_match:
+            if in_ol:
+                html_lines.append("</ol>")
+                in_ol = False
+            if not in_ul:
+                html_lines.append("<ul>")
+                in_ul = True
+            html_lines.append(f"<li>{render_inline(ul_match.group(1))}</li>")
+            continue
+
+        ol_match = re.match(r"^\s*\d+\.\s+(.*)$", line)
+        if ol_match:
+            if in_ul:
+                html_lines.append("</ul>")
+                in_ul = False
+            if not in_ol:
+                html_lines.append("<ol>")
+                in_ol = True
+            html_lines.append(f"<li>{render_inline(ol_match.group(1))}</li>")
+            continue
+
+        close_lists()
+
+        # Inline conversions (leave raw HTML intact)
+        if stripped == "[TOC]":
+            toc_items = []
+            for level, title, slug in headings:
+                toc_items.append(
+                    f'<li class="toc-level-{level}"><a href="#{slug}">{title}</a></li>',
+                )
+            html_lines.append(
+                '<nav class="toc"><ul>' + "".join(toc_items) + "</ul></nav>",
+            )
+            continue
+
+        html_lines.append(f"<p>{render_inline(line)}</p>")
+
+    close_lists()
+    if in_blockquote:
+        html_lines.append("</blockquote>")
+    if in_code:
+        html_lines.append("</code></pre>")
+
+    return "\n".join(html_lines)
+
+
+def _render_markdown_document(markdown_text: str) -> str:
+    body = _render_markdown_fallback(markdown_text)
+    wrapped = (
+        '<!doctype html><html><head><meta charset="utf-8">'
+        '<meta name="viewport" content="width=device-width,initial-scale=1">'
+        "<style>body{max-width:900px;margin:24px auto;padding:0 16px;"
+        "font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif;"
+        "line-height:1.55;} img{max-width:100%;} pre{background:#f6f6f6;padding:12px;overflow:auto;}"
+        ".toc ul{list-style:none;padding-left:0;} .toc li{margin:4px 0;}</style>"
+        "</head><body>"
+        f"{body}"
+        "</body></html>"
+    )
+    return wrapped
+
+
 def _content_type_base(content_type: str) -> str:
     return (content_type or "").split(";", 1)[0].strip().lower()
 
@@ -369,7 +859,6 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
     if config is None:
         config = get_config(resolve_plugins=False)
     fullpath, path = _resolve_archive_path(document_root, path)
-    replay_response = partial(_apply_archive_replay_headers, fullpath=fullpath, is_archive_replay=is_archive_replay, config=config)
     if os.access(fullpath, os.R_OK) and fullpath.is_dir():
         if request.GET.get("download") == "zip" and show_indexes:
             return _build_directory_zip_response(
@@ -383,7 +872,13 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             response = _render_directory_index(request, path, fullpath)
             response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
             response.headers["Last-Modified"] = http_date(fullpath.stat().st_mtime)
-            return replay_response(response, content_type="text/html")
+            return _apply_archive_replay_headers(
+                response,
+                fullpath=fullpath,
+                content_type="text/html",
+                is_archive_replay=is_archive_replay,
+                config=config,
+            )
         raise Http404(_("Directory indexes are not allowed here."))
     if not os.access(fullpath, os.R_OK):
         raise Http404(_("“%(path)s” does not exist") % {"path": fullpath})
@@ -406,7 +901,13 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 not_modified.headers["ETag"] = etag
                 not_modified.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable"
                 not_modified.headers["Last-Modified"] = http_date(statobj.st_mtime)
-                return replay_response(not_modified, content_type="")
+                return _apply_archive_replay_headers(
+                    not_modified,
+                    fullpath=fullpath,
+                    content_type="",
+                    is_archive_replay=is_archive_replay,
+                    config=config,
+                )
 
     content_type, encoding = mimetypes.guess_type(str(fullpath))
     preserve_plain_text = fullpath.suffix.lower() in {".log", ".sh"}
@@ -439,12 +940,13 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
         request.META.get("HTTP_IF_MODIFIED_SINCE"),
         statobj.st_mtime,
     ):
-        return replay_response(HttpResponseNotModified(), content_type=content_type)
-
-    def transformed_response(body: str, response_type: str):
-        response = HttpResponse(body, content_type=response_type)
-        _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
-        return replay_response(response, content_type=response_type)
+        return _apply_archive_replay_headers(
+            HttpResponseNotModified(),
+            fullpath=fullpath,
+            content_type=content_type,
+            is_archive_replay=is_archive_replay,
+            config=config,
+        )
 
     # Wrap text-like outputs in HTML when explicitly requested for iframe previewing.
     if preview_as_text_html:
@@ -452,8 +954,16 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             max_preview_size = 10 * 1024 * 1024
             if statobj.st_size <= max_preview_size:
                 decoded = fullpath.read_text(encoding="utf-8", errors="replace")
-                wrapped = replay_preview._render_text_preview_document(decoded, fullpath.name)
-                return transformed_response(wrapped, "text/html; charset=utf-8")
+                wrapped = _render_text_preview_document(decoded, fullpath.name)
+                response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
+                _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
+                return _apply_archive_replay_headers(
+                    response,
+                    fullpath=fullpath,
+                    content_type="text/html; charset=utf-8",
+                    is_archive_replay=is_archive_replay,
+                    config=config,
+                )
         except (OSError, UnicodeDecodeError, ValueError):
             preview_as_text_html = False
 
@@ -464,8 +974,16 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             raw_image_url = request.path
             if preview_query:
                 raw_image_url = f"{raw_image_url}?{urlencode(list(preview_query.lists()), doseq=True)}"
-            wrapped = replay_preview._render_image_preview_document(raw_image_url, fullpath.name)
-            return transformed_response(wrapped, "text/html; charset=utf-8")
+            wrapped = _render_image_preview_document(raw_image_url, fullpath.name)
+            response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
+            _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
+            return _apply_archive_replay_headers(
+                response,
+                fullpath=fullpath,
+                content_type="text/html; charset=utf-8",
+                is_archive_replay=is_archive_replay,
+                config=config,
+            )
         except (OSError, ValueError):
             preview_as_image_html = False
 
@@ -493,7 +1011,13 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             response = HttpResponse(body, content_type=preview_content_type)
             for key, value in headers.items():
                 response.headers[key] = value
-            return replay_response(response, content_type=preview_content_type)
+            return _apply_archive_replay_headers(
+                response,
+                fullpath=fullpath,
+                content_type=preview_content_type,
+                is_archive_replay=is_archive_replay,
+                config=config,
+            )
         except (OSError, RuntimeError, ValueError):
             preview_as_archivewebpage_html = False
 
@@ -511,37 +1035,52 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                     decoded = html.unescape(decoded)
                 rewritten_html, rewritten_count = ("", 0)
                 if content_type.startswith("text/html") and document_root:
-                    rewritten_html, rewritten_count = replay_preview._rewrite_html_image_sources_for_request(
-                        request,
-                        decoded,
-                        document_root,
-                        rel_path,
+                    rewritten_html, rewritten_count = _rewrite_html_image_sources_for_request(request, decoded, document_root, rel_path)
+                markdown_candidate = _extract_markdown_candidate(decoded)
+                if _looks_like_markdown(markdown_candidate):
+                    wrapped = _render_markdown_document(markdown_candidate)
+                    wrapped, _rewrite_count = _rewrite_html_image_sources_for_request(request, wrapped, document_root, rel_path)
+                    wrapped = _apply_transformed_html_preview_style(wrapped)
+                    response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
+                    _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
+                    return _apply_archive_replay_headers(
+                        response,
+                        fullpath=fullpath,
+                        content_type="text/html; charset=utf-8",
+                        is_archive_replay=is_archive_replay,
+                        config=config,
                     )
-                markdown_candidate = replay_preview._extract_markdown_candidate(decoded)
-                if replay_preview._looks_like_markdown(markdown_candidate):
-                    wrapped = replay_preview._render_markdown_document(markdown_candidate)
-                    wrapped, _rewrite_count = replay_preview._rewrite_html_image_sources_for_request(
-                        request,
-                        wrapped,
-                        document_root,
-                        rel_path,
-                    )
-                    wrapped = replay_preview._apply_transformed_html_preview_style(wrapped)
-                    return transformed_response(wrapped, "text/html; charset=utf-8")
                 if rewritten_count:
-                    rewritten_html = replay_preview._apply_transformed_html_preview_style(rewritten_html)
-                    return transformed_response(rewritten_html, content_type)
+                    rewritten_html = _apply_transformed_html_preview_style(rewritten_html)
+                    response = HttpResponse(rewritten_html, content_type=content_type)
+                    _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
+                    return _apply_archive_replay_headers(
+                        response,
+                        fullpath=fullpath,
+                        content_type=content_type,
+                        is_archive_replay=is_archive_replay,
+                        config=config,
+                    )
                 if escaped_count and escaped_count > tag_count * 2:
-                    decoded = replay_preview._apply_transformed_html_preview_style(decoded)
-                    return transformed_response(decoded, content_type)
+                    decoded = _apply_transformed_html_preview_style(decoded)
+                    response = HttpResponse(decoded, content_type=content_type)
+                    _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
+                    return _apply_archive_replay_headers(
+                        response,
+                        fullpath=fullpath,
+                        content_type=content_type,
+                        is_archive_replay=is_archive_replay,
+                        config=config,
+                    )
         except (OSError, UnicodeDecodeError, ValueError):
             pass
 
     # setup response object
     ranged_file = RangedFileReader(fullpath.open("rb"))
-    response = StreamingHttpResponse(ranged_file, content_type=content_type)
-    if _is_asgi_request(request):
-        response.streaming_content = _stream_async(ranged_file)
+    response = StreamingHttpResponse(
+        _stream_ranged_file_async(ranged_file) if _is_asgi_request(request) else ranged_file,
+        content_type=content_type,
+    )
     response.headers["Last-Modified"] = http_date(statobj.st_mtime)
     if etag:
         response.headers["ETag"] = etag
@@ -579,7 +1118,13 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 response.status_code = 206
     if encoding:
         response.headers["Content-Encoding"] = encoding
-    return replay_response(response, content_type=content_type)
+    return _apply_archive_replay_headers(
+        response,
+        fullpath=fullpath,
+        content_type=content_type,
+        is_archive_replay=is_archive_replay,
+        config=config,
+    )
 
 
 def serve_static(request, path, **kwargs):

archivebox/misc/toml_util.py

@@ -2,11 +2,94 @@ from typing import Any, cast
 from collections.abc import Callable
 
 import json
+import ast
+import inspect
 import toml
 import re
+import configparser
 
 from pathlib import Path, PosixPath
 
+from pydantic.json_schema import GenerateJsonSchema
+from pydantic_core import to_jsonable_python
+
+JSONValue = str | bool | int | None | list["JSONValue"]
+
+TOML_HEADER = "# Converted from INI to TOML format: https://toml.io/en/\n\n"
+
+
+def load_ini_value(val: str) -> JSONValue:
+    """Convert lax INI values into strict TOML-compliant (JSON) values"""
+    if val.lower() in ("true", "yes", "1"):
+        return True
+    if val.lower() in ("false", "no", "0"):
+        return False
+    if val.isdigit():
+        return int(val)
+
+    try:
+        return ast.literal_eval(val)
+    except Exception:
+        pass
+
+    try:
+        return json.loads(val)
+    except Exception:
+        pass
+
+    return val
+
+
+def convert(ini_str: str) -> str:
+    """Convert a string of INI config into its TOML equivalent (warning: strips comments)"""
+
+    config = configparser.ConfigParser()
+    setattr(config, "optionxform", str)  # capitalize key names
+    config.read_string(ini_str)
+
+    # Initialize an empty dictionary to store the TOML representation
+    toml_dict = {}
+
+    # Iterate over each section in the INI configuration
+    for section in config.sections():
+        toml_dict[section] = {}
+
+        # Iterate over each key-value pair in the section
+        for key, value in config.items(section):
+            parsed_value = load_ini_value(value)
+
+            # Convert the parsed value to its TOML-compatible JSON representation
+            toml_dict[section.upper()][key.upper()] = json.dumps(parsed_value)
+
+    # Build the TOML string
+    toml_str = TOML_HEADER
+    for section, items in toml_dict.items():
+        toml_str += f"[{section}]\n"
+        for key, value in items.items():
+            toml_str += f"{key} = {value}\n"
+        toml_str += "\n"
+
+    return toml_str.strip()
+
+
+class JSONSchemaWithLambdas(GenerateJsonSchema):
+    """
+    Encode lambda functions in default values properly.
+    Usage:
+    >>> json.dumps(value, encoder=JSONSchemaWithLambdas())
+    """
+
+    def encode_default(self, dft: Any) -> Any:
+        config = self._config
+        if isinstance(dft, Callable):
+            return "{{lambda " + inspect.getsource(dft).split("=lambda ")[-1].strip()[:-1] + "}}"
+        return to_jsonable_python(
+            dft,
+            timedelta_mode=config.ser_json_timedelta,
+            bytes_mode=config.ser_json_bytes,
+            serialize_unknown=True,
+        )
+
 
 def better_toml_dump_str(val: Any) -> str:
     try:

archivebox/personas/admin.py

@@ -5,7 +5,7 @@ import shutil
 from django.contrib import admin, messages
 from django.utils.html import format_html, format_html_join
 
-from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
+from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 from archivebox.personas.forms import PersonaAdminForm
 from archivebox.personas.importers import discover_local_browser_profiles
 from archivebox.personas.models import Persona
@@ -30,20 +30,28 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
                 "classes": ("card", "persona-card-primary"),
             },
         ),
-        card_fieldset(
+        (
             "Browser Import",
-            (
-                "import_mode",
-                "import_discovered_profile",
-                "import_source",
-                "import_profile_name",
-                "import_copy_profile",
-                "import_extract_cookies",
-                "import_capture_storage",
-            ),
-            wide=True,
+            {
+                "fields": (
+                    "import_mode",
+                    "import_discovered_profile",
+                    "import_source",
+                    "import_profile_name",
+                    "import_copy_profile",
+                    "import_extract_cookies",
+                    "import_capture_storage",
+                ),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Advanced",
+            {
+                "fields": ("config",),
+                "classes": ("card", "wide"),
+            },
         ),
-        card_fieldset("Advanced", ("config",), wide=True),
     )
 
     change_fieldsets = (
@@ -57,7 +65,13 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
         ),
         add_fieldsets[1],
         add_fieldsets[2],
-        card_fieldset("Artifacts", ("persona_paths", "import_artifact_status"), wide=True),
+        (
+            "Artifacts",
+            {
+                "fields": ("persona_paths", "import_artifact_status"),
+                "classes": ("card", "wide"),
+            },
+        ),
     )
 
     @admin.display(description="Chrome Profile")

archivebox/personas/importers.py

@@ -216,53 +216,118 @@ class PersonaImportResult:
         return self.profile_copied or self.cookies_imported or self.storage_captured or self.user_agent_imported
 
 
-# Relative profile roots in lookup order. Chromium additionally supports XDG
-# and snap; the other browsers retain their own conventional config locations.
-BROWSER_PROFILE_PATHS = {
-    "chrome": {
-        "Darwin": ("Google/Chrome",),
-        "Linux": ("google-chrome", "chrome"),
-        "Windows": ("Google/Chrome/User Data",),
-    },
-    "chromium": {
-        "Darwin": ("Chromium",),
-        "Linux": ("chromium",),
-        "Windows": ("Chromium/User Data",),
-    },
-    "brave": {
-        "Darwin": ("BraveSoftware/Brave-Browser",),
-        "Linux": ("BraveSoftware/Brave-Browser",),
-        "Windows": ("BraveSoftware/Brave-Browser/User Data",),
-    },
-    "edge": {
-        "Darwin": ("Microsoft Edge",),
-        "Linux": ("microsoft-edge", "microsoft-edge-beta", "microsoft-edge-dev"),
-        "Windows": ("Microsoft/Edge/User Data",),
-    },
-}
-CHROMIUM_BROWSERS = tuple(BROWSER_PROFILE_PATHS)
-
-
-def get_browser_user_data_dir(browser: str) -> Path | None:
-    """Find a browser's first installed profile root on the current platform."""
-    home = Path.home()
+def get_chrome_user_data_dir() -> Path | None:
+    """Get the default Chrome user data directory for the current platform."""
     system = platform.system()
-    # Chromium historically uses Linux locations on other Unix platforms too.
-    if browser == "chromium" and system not in ("Darwin", "Windows"):
-        system = "Linux"
-    roots = {
-        "Darwin": home / "Library/Application Support",
-        "Linux": Path(os.environ.get("XDG_CONFIG_HOME", home / ".config")) if browser == "chromium" else home / ".config",
-        "Windows": Path(os.environ.get("LOCALAPPDATA", home / "AppData/Local")),
-    }
-    if system not in roots:
-        return None
-    candidates = [roots[system] / relative for relative in BROWSER_PROFILE_PATHS[browser].get(system, ())]
-    if browser == "chromium" and system == "Linux":
-        candidates.append(home / "snap/chromium/common/chromium")
+    home = Path.home()
+
+    if system == "Darwin":
+        candidates = [
+            home / "Library" / "Application Support" / "Google" / "Chrome",
+        ]
+    elif system == "Linux":
+        candidates = [
+            home / ".config" / "google-chrome",
+            home / ".config" / "chrome",
+        ]
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = [
+            local_app_data / "Google" / "Chrome" / "User Data",
+        ]
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate.exists() and _list_profile_names(candidate):
+            return candidate
+
+    return None
+
+
+def get_chromium_user_data_dir() -> Path | None:
+    """Select Chromium's own profile even when Google Chrome is installed."""
+    home = Path.home()
+    if platform.system() == "Darwin":
+        candidates = [home / "Library/Application Support/Chromium"]
+    elif platform.system() == "Windows":
+        candidates = [Path(os.environ.get("LOCALAPPDATA", home / "AppData/Local")) / "Chromium/User Data"]
+    else:
+        candidates = [
+            Path(os.environ.get("XDG_CONFIG_HOME", home / ".config")) / "chromium",
+            home / "snap/chromium/common/chromium",
+        ]
     return next((path for path in candidates if path.is_dir() and _list_profile_names(path)), None)
 
 
+def get_brave_user_data_dir() -> Path | None:
+    """Get the default Brave user data directory for the current platform."""
+    system = platform.system()
+    home = Path.home()
+
+    if system == "Darwin":
+        candidates = [
+            home / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser",
+        ]
+    elif system == "Linux":
+        candidates = [
+            home / ".config" / "BraveSoftware" / "Brave-Browser",
+        ]
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = [
+            local_app_data / "BraveSoftware" / "Brave-Browser" / "User Data",
+        ]
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate.exists() and _list_profile_names(candidate):
+            return candidate
+
+    return None
+
+
+def get_edge_user_data_dir() -> Path | None:
+    """Get the default Edge user data directory for the current platform."""
+    system = platform.system()
+    home = Path.home()
+
+    if system == "Darwin":
+        candidates = [
+            home / "Library" / "Application Support" / "Microsoft Edge",
+        ]
+    elif system == "Linux":
+        candidates = [
+            home / ".config" / "microsoft-edge",
+            home / ".config" / "microsoft-edge-beta",
+            home / ".config" / "microsoft-edge-dev",
+        ]
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = [
+            local_app_data / "Microsoft" / "Edge" / "User Data",
+        ]
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate.exists() and _list_profile_names(candidate):
+            return candidate
+
+    return None
+
+
+BROWSER_PROFILE_FINDERS = {
+    "chrome": get_chrome_user_data_dir,
+    "chromium": get_chromium_user_data_dir,
+    "brave": get_brave_user_data_dir,
+    "edge": get_edge_user_data_dir,
+}
+
+CHROMIUM_BROWSERS = tuple(BROWSER_PROFILE_FINDERS.keys())
+
+
 NETSCAPE_COOKIE_HEADER = [
     "# Netscape HTTP Cookie File",
     "# https://curl.se/docs/http-cookies.html",
@@ -291,8 +356,8 @@ def validate_persona_name(name: str) -> tuple[bool, str]:
 def discover_local_browser_profiles() -> list[PersonaImportSource]:
     discovered: list[PersonaImportSource] = []
 
-    for browser in CHROMIUM_BROWSERS:
-        user_data_dir = get_browser_user_data_dir(browser)
+    for browser, finder in BROWSER_PROFILE_FINDERS.items():
+        user_data_dir = finder()
         if not user_data_dir:
             continue
 
@@ -365,11 +430,11 @@ def discover_persona_template_profiles(personas_dir: Path | None = None) -> list
 
 def resolve_browser_import_source(browser: str, profile_dir: str | None = None) -> PersonaImportSource:
     browser = browser.lower().strip()
-    if browser not in CHROMIUM_BROWSERS:
-        supported = ", ".join(CHROMIUM_BROWSERS)
+    if browser not in BROWSER_PROFILE_FINDERS:
+        supported = ", ".join(BROWSER_PROFILE_FINDERS)
         raise ValueError(f"Unknown browser: {browser}. Supported browsers: {supported}")
 
-    user_data_dir = get_browser_user_data_dir(browser)
+    user_data_dir = BROWSER_PROFILE_FINDERS[browser]()
     if not user_data_dir:
         raise ValueError(
             f"Could not find {browser} profile directory. Use --source /path/to/browser-data. "

archivebox/personas/models.py

@@ -131,16 +131,6 @@ class Persona(ModelWithConfig):
     def __str__(self) -> str:
         return self.name
 
-    def to_json(self) -> dict:
-        """Portable CLI record, shared by create, list, and update."""
-        return {
-            "id": str(self.id),
-            "name": self.name,
-            "path": str(self.path),
-            "CHROME_USER_DATA_DIR": self.CHROME_USER_DATA_DIR,
-            "COOKIES_FILE": self.COOKIES_FILE,
-        }
-
     @property
     def path(self) -> Path:
         """Path to persona directory under PERSONAS_DIR."""
@@ -299,54 +289,87 @@ class Persona(ModelWithConfig):
 
     def prepare_runtime_for_crawl(self, crawl, chrome_binary: str = "") -> dict[str, str]:
         self.ensure_dirs()
+
+        template_dir = Path(self.CHROME_USER_DATA_DIR)
+        runtime_root = self.runtime_root_for_crawl(crawl)
+        runtime_profile_dir = self.runtime_profile_dir_for_crawl(crawl)
+        runtime_downloads_dir = self.runtime_downloads_dir_for_crawl(crawl)
+
         with self.lock_runtime_for_crawl():
-            return self._prepare_runtime(
-                self.runtime_root_for_crawl(crawl),
-                Path(self.CHROME_USER_DATA_DIR),
-                (self.path,),
-                chrome_binary,
-            )
+            if runtime_root.exists():
+                shutil.rmtree(runtime_root, ignore_errors=True)
+            if template_dir.exists() and any(template_dir.iterdir()):
+                self.copy_chrome_profile(template_dir, runtime_profile_dir)
+            else:
+                runtime_profile_dir.mkdir(parents=True, exist_ok=True)
+
+            for filename in ("cookies.txt", "auth.json"):
+                source = self.path / filename
+                if source.is_file():
+                    shutil.copy2(source, runtime_root / filename)
+            runtime_downloads_dir.mkdir(parents=True, exist_ok=True)
+            self.cleanup_chrome_profile(runtime_profile_dir)
+
+            (runtime_root / "persona_name.txt").write_text(self.name)
+            (runtime_root / "template_dir.txt").write_text(str(template_dir))
+            if chrome_binary:
+                (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
+
+        return {
+            # Hooks derive CHROME_USER_DATA_DIR/CHROME_DOWNLOADS_DIR from
+            # PERSONAS_DIR + ACTIVE_PERSONA. Point PERSONAS_DIR at the
+            # per-crawl runtime root here so CHROME_ISOLATION=crawl never
+            # leaks or reuses the template profile while keeping Chrome path
+            # derivation centralized in the Chrome plugin helpers.
+            "PERSONAS_DIR": str(runtime_root.parent),
+            "ACTIVE_PERSONA": self.name,
+            **{
+                key: str(runtime_root / filename)
+                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
+                if (runtime_root / filename).is_file()
+            },
+        }
 
     def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
-        crawl_profile = self.runtime_profile_dir_for_crawl(snapshot.crawl)
-        return self._prepare_runtime(
-            self.runtime_root_for_snapshot(snapshot),
-            crawl_profile if crawl_profile.exists() else Path(self.CHROME_USER_DATA_DIR),
-            (self.runtime_root_for_crawl(snapshot.crawl), self.path),
-            chrome_binary,
-        )
+        crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
+        template_dir = crawl_runtime_profile_dir if crawl_runtime_profile_dir.exists() else Path(self.CHROME_USER_DATA_DIR)
+        runtime_root = self.runtime_root_for_snapshot(snapshot)
+        runtime_profile_dir = self.runtime_profile_dir_for_snapshot(snapshot)
+        runtime_downloads_dir = self.runtime_downloads_dir_for_snapshot(snapshot)
 
-    def _prepare_runtime(
-        self,
-        runtime_root: Path,
-        template_dir: Path,
-        auth_sources: tuple[Path, ...],
-        chrome_binary: str,
-    ) -> dict[str, str]:
-        """Clone one isolated profile, taking each auth file from its first available source."""
-        profile_dir = runtime_root / "chrome_profile"
-        shutil.rmtree(runtime_root, ignore_errors=True)
+        if runtime_root.exists():
+            shutil.rmtree(runtime_root, ignore_errors=True)
         if template_dir.exists() and any(template_dir.iterdir()):
-            self.copy_chrome_profile(template_dir, profile_dir)
+            self.copy_chrome_profile(template_dir, runtime_profile_dir)
         else:
-            profile_dir.mkdir(parents=True, exist_ok=True)
+            runtime_profile_dir.mkdir(parents=True, exist_ok=True)
+
+        for filename in ("cookies.txt", "auth.json"):
+            source = self.runtime_root_for_crawl(snapshot.crawl) / filename
+            if not source.is_file():
+                source = self.path / filename
+            if source.is_file():
+                shutil.copy2(source, runtime_root / filename)
+        runtime_downloads_dir.mkdir(parents=True, exist_ok=True)
+        self.cleanup_chrome_profile(runtime_profile_dir)
 
-        overrides = {"PERSONAS_DIR": str(runtime_root.parent), "ACTIVE_PERSONA": self.name}
-        for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json")):
-            source = next((root / filename for root in auth_sources if (root / filename).is_file()), None)
-            if source is not None:
-                destination = runtime_root / filename
-                shutil.copy2(source, destination)
-                overrides[key] = str(destination)
-        (runtime_root / "chrome_downloads").mkdir(parents=True, exist_ok=True)
-        self.cleanup_chrome_profile(profile_dir)
         (runtime_root / "persona_name.txt").write_text(self.name)
         (runtime_root / "template_dir.txt").write_text(str(template_dir))
         if chrome_binary:
             (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        # Chrome hooks derive their directories from this isolated persona root.
-        return overrides
+        return {
+            # See prepare_runtime_for_crawl(): snapshot isolation changes the
+            # persona root, not individual CHROME_* config keys, so standalone
+            # Chrome hooks and ArchiveBox-driven hooks resolve paths the same way.
+            "PERSONAS_DIR": str(runtime_root.parent),
+            "ACTIVE_PERSONA": self.name,
+            **{
+                key: str(runtime_root / filename)
+                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
+                if (runtime_root / filename).is_file()
+            },
+        }
 
     def cleanup_runtime_for_crawl(self, crawl) -> None:
         shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True)

archivebox/plugins/discovery.py

@@ -55,6 +55,10 @@ def get_plugins() -> list[str]:
     return sorted(get_plugin_catalog())
 
 
+def get_plugin_models():
+    return get_plugin_catalog().plugins
+
+
 def get_plugin_name(plugin: str) -> str:
     """
     Get the base plugin name without numeric prefix.

archivebox/plugins/forms.py

@@ -87,14 +87,23 @@ def _coerce_plugin_config_value(raw_value: Any, schema: Mapping[str, Any]) -> An
             return False
         raise forms.ValidationError("Must be true or false.")
 
-    if "integer" in schema_types or "number" in schema_types:
-        convert = int if "integer" in schema_types else float
-        value = convert(str(raw_value).strip())
+    if "integer" in schema_types:
+        value = int(str(raw_value).strip())
         minimum = schema.get("minimum")
         maximum = schema.get("maximum")
-        if minimum is not None and value < convert(minimum):
+        if minimum is not None and value < int(minimum):
             raise forms.ValidationError(f"Must be at least {minimum}.")
-        if maximum is not None and value > convert(maximum):
+        if maximum is not None and value > int(maximum):
+            raise forms.ValidationError(f"Must be at most {maximum}.")
+        return value
+
+    if "number" in schema_types:
+        value = float(str(raw_value).strip())
+        minimum = schema.get("minimum")
+        maximum = schema.get("maximum")
+        if minimum is not None and value < float(minimum):
+            raise forms.ValidationError(f"Must be at least {minimum}.")
+        if maximum is not None and value > float(maximum):
             raise forms.ValidationError(f"Must be at most {maximum}.")
         return value
 
@@ -139,24 +148,28 @@ class PluginConfigFormMixin:
         self.plugin_config_binary_urls = get_plugin_config_binary_urls(runtime_config)
         grouped_plugins = catalog.groups()
 
-        binary_url_lookup = _build_required_binary_url_lookup(plugin_configs, runtime_config)
-        self.plugin_groups = []
+        group_specs = []
         for category, field_name, title in PLUGIN_GROUPS:
-            plugin_names = tuple(plugin.name for plugin in grouped_plugins.get(category, []) if plugin.name in all_plugins)
+            plugin_names = tuple(plugin.name for plugin in grouped_plugins.get(category, []))
+            group_specs.append((field_name, title, "", "", "", plugin_names))
             if field_name in self.fields:
-                get_choice_field(self, field_name).choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names]
-            if plugin_names:
-                self.plugin_groups.append(
-                    {
-                        "field_name": field_name,
-                        "title": title,
-                        "note": "",
-                        "dom_id": "",
-                        "select_all_group": "",
-                        "show_selectors": field_name in self.fields,
-                        "plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config, binary_url_lookup),
-                    },
-                )
+                get_choice_field(self, field_name).choices = [
+                    (p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins
+                ]
+        binary_url_lookup = _build_required_binary_url_lookup(plugin_configs, runtime_config)
+        self.plugin_groups = [
+            {
+                "field_name": field_name,
+                "title": title,
+                "note": note,
+                "dom_id": dom_id,
+                "select_all_group": select_all_group,
+                "show_selectors": field_name in self.fields,
+                "plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config, binary_url_lookup),
+            }
+            for field_name, title, note, dom_id, select_all_group, plugin_names in group_specs
+            if any(plugin in all_plugins for plugin in plugin_names)
+        ]
 
     def _build_plugin_cards(
         self,

archivebox/progressmonitor/collection.py

@@ -0,0 +1,54 @@
+"""Non-blocking, database-only collection totals for the progress monitor."""
+
+import logging
+from threading import Lock, Thread
+from time import time
+
+from django.core.cache import cache
+from django.db import close_old_connections, connections
+from django.db.models import Count, Sum
+
+
+_refresh_lock = Lock()
+
+
+def collection_summary(user):
+    user_id = None if user.is_superuser else user.pk
+    key = f"progress-collection:{'all' if user_id is None else user_id}"
+    summary = cache.get(key)
+    # Even an indexed SUM scans the index. Never make an HTTP request wait for
+    # that scan on a large collection. Serve stale totals while one worker refreshes;
+    # a single non-blocking lock prevents a queue of scans from concurrent clients.
+    if (summary is None or time() - summary["sampled_at"] >= 60) and _refresh_lock.acquire(blocking=False):
+        # Throttle failures too: a broken/locked database must not trigger work
+        # and log output on every sidebar poll. This cache is local to each worker.
+        if not cache.add(f"{key}:refresh", True, timeout=60):
+            _refresh_lock.release()
+            return summary
+
+        def refresh():
+            try:
+                close_old_connections()
+                from archivebox.core.models import Snapshot
+
+                snapshots = Snapshot.objects.all()
+                if user_id is not None:
+                    snapshots = snapshots.filter(crawl__created_by_id=user_id)
+                # COUNT(*) + SUM(output_size) can use Snapshot's covering size index.
+                # Snapshot.output_size is maintained when ArchiveResult sizes change.
+                totals = snapshots.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
+                totals["bytes"] = totals["bytes"] or 0
+                totals["sampled_at"] = time()
+                cache.set(key, totals, timeout=3600)
+            except Exception:
+                logging.getLogger(__name__).exception("Could not refresh collection totals")
+            finally:
+                connections.close_all()
+                _refresh_lock.release()
+
+        try:
+            Thread(target=refresh, name="collection-summary", daemon=True).start()
+        except RuntimeError:
+            _refresh_lock.release()
+            logging.getLogger(__name__).exception("Could not start collection totals refresh")
+    return summary

archivebox/progressmonitor/presentation.py

@@ -1,432 +0,0 @@
-"""Render the monitor's crawl, snapshot, and hook payloads from loaded data."""
-
-from pathlib import Path
-from archivebox.config.constants import CONSTANTS
-from archivebox.config.common import get_config
-from archivebox.core.models import ArchiveResult, Snapshot
-from archivebox.crawls.models import Crawl
-from archivebox.core.routes_util import build_snapshot_url, build_web_url
-from archivebox.misc.logging_util import printable_filesize
-
-
-def render_crawl(report, crawl):
-    crawl_id = str(crawl["id"])
-    crawl_snapshot_counts = report.snapshot_counts_by_crawl.get(crawl_id, {})
-    total_snapshots = sum(crawl_snapshot_counts.values())
-    completed_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.SEALED, 0)
-    started_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.STARTED, 0)
-    pending_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.QUEUED, 0)
-    cancelled_snapshots = report.cancelled_snapshot_counts_by_crawl.get(crawl_id, 0)
-
-    # Count URLs in the crawl (for when snapshots haven't been created yet)
-    urls_count = 0
-    if crawl["urls"]:
-        urls_count = len([u for u in crawl["urls"].split("\n") if u.strip() and not u.startswith("#")])
-
-    # Calculate crawl progress
-    crawl_progress = int((completed_snapshots / total_snapshots) * 100) if total_snapshots > 0 else 0
-    crawl_run_started_at = crawl["created_at"]
-    crawl_setup_plugins = [
-        payload
-        for payload, proc_started_at in report.process_records_by_crawl.get(crawl_id, [])
-        if is_current_run_timestamp(proc_started_at, crawl_run_started_at)
-    ]
-    crawl_setup_total = len(crawl_setup_plugins)
-    crawl_setup_completed = sum(1 for item in crawl_setup_plugins if item.get("status") == "succeeded")
-    crawl_setup_failed = sum(1 for item in crawl_setup_plugins if item.get("status") == "failed")
-    crawl_setup_pending = sum(1 for item in crawl_setup_plugins if item.get("status") == "queued")
-    crawl_screencast_url = screencast_frame_url(report, crawl_id, report.active_crawl_objects[crawl_id].output_dir)
-    crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id.replace('-', '')}/change/" if crawl_screencast_url else ""
-
-    active_snapshots_for_crawl = [
-        render_snapshot(report, snapshot, crawl_id) for snapshot in report.displayed_snapshots_by_crawl.get(crawl_id, [])
-    ]
-
-    # Check if crawl can start (for debugging stuck crawls)
-    can_start = bool(crawl["urls"])
-    urls_preview = crawl["urls"][:60] if crawl["urls"] else None
-    crawl_tags = [tag.strip() for tag in (crawl["tags_str"] or "").replace("\n", ",").split(",") if tag.strip()]
-    persona_details = report.persona_details_by_id.get(str(crawl["persona_id"])) if crawl["persona_id"] else None
-    persona_name = persona_details["name"] if persona_details else "Default"
-    persona_details = persona_details or report.persona_details_by_name.get(persona_name)
-    crawl_output_size = report.crawl_output_sizes_by_crawl.get(crawl_id, 0)
-    avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0
-    crawl_obj = report.active_crawl_objects[crawl_id]
-    effective_crawl_config = get_config(crawl=crawl_obj, resolve_plugins=False)
-    max_urls = int(effective_crawl_config.CRAWL_MAX_URLS or 0)
-    crawl_max_size = int(effective_crawl_config.CRAWL_MAX_SIZE or 0)
-    crawl_timeout = int(effective_crawl_config.CRAWL_TIMEOUT or 0)
-    snapshot_max_size = int(effective_crawl_config.SNAPSHOT_MAX_SIZE or 0)
-
-    # Check if retry_at is in the future (would prevent worker from claiming)
-    retry_at_future = crawl["retry_at"] > report.now if crawl["retry_at"] else False
-    is_paused = crawl_obj.is_paused
-    seconds_until_retry = (
-        0 if is_paused else int((crawl["retry_at"] - report.now).total_seconds()) if crawl["retry_at"] and retry_at_future else 0
-    )
-    crawl_worker_state = (
-        "running"
-        if report.crawl_process_pids.get(crawl_id)
-        or any(isinstance(snapshot, dict) and snapshot.get("worker_pid") for snapshot in active_snapshots_for_crawl)
-        else "waiting"
-    )
-    if is_paused:
-        crawl_worker_state = "paused"
-    elif crawl["status"] == Crawl.StatusChoices.STARTED and crawl_worker_state == "waiting" and (started_snapshots or pending_snapshots):
-        crawl_worker_state = "waiting" if report.orchestrator_running else "crashed"
-
-    return {
-        "id": crawl_id,
-        "label": (next((line.strip() for line in (crawl["urls"] or "").splitlines() if line.strip()), "") or crawl_id)[:60],
-        "status": crawl["status"],
-        "is_paused": is_paused,
-        "started": crawl["created_at"].isoformat() if crawl["created_at"] else None,
-        "progress": crawl_progress,
-        "created_by": crawl["created_by__username"],
-        "persona": persona_name,
-        "persona_admin_url": persona_details["admin_url"] if persona_details else None,
-        "max_depth": crawl["max_depth"],
-        "max_urls": max_urls,
-        "max_crawl_size": crawl_max_size,
-        "crawl_timeout": crawl_timeout,
-        "max_snapshot_size": snapshot_max_size,
-        "max_crawl_size_display": printable_filesize(crawl_max_size) if crawl_max_size else "unlimited",
-        "crawl_timeout_display": f"{crawl_timeout}s" if crawl_timeout else "unlimited",
-        "max_snapshot_size_display": printable_filesize(snapshot_max_size) if snapshot_max_size else "unlimited",
-        "crawl_output_size": crawl_output_size,
-        "avg_snapshot_size": avg_snapshot_size,
-        "crawl_output_size_display": printable_filesize(crawl_output_size) if crawl_output_size else "0 B",
-        "avg_snapshot_size_display": printable_filesize(avg_snapshot_size) if avg_snapshot_size else "0 B",
-        "tags": crawl_tags,
-        "urls_count": urls_count,
-        "total_snapshots": total_snapshots,
-        "completed_snapshots": completed_snapshots,
-        "started_snapshots": started_snapshots,
-        "failed_snapshots": 0,
-        "pending_snapshots": pending_snapshots,
-        "cancelled_snapshots": cancelled_snapshots,
-        "setup_plugins": crawl_setup_plugins,
-        "setup_total_plugins": crawl_setup_total,
-        "setup_completed_plugins": crawl_setup_completed,
-        "setup_failed_plugins": crawl_setup_failed,
-        "setup_pending_plugins": crawl_setup_pending,
-        "screencast_url": crawl_screencast_url,
-        "screencast_link": crawl_screencast_link,
-        "active_snapshots": active_snapshots_for_crawl,
-        "queued_snapshots_hidden": report.queued_snapshot_overflow_by_crawl.get(crawl_id, 0),
-        "can_start": can_start,
-        "urls_preview": urls_preview,
-        "retry_at_future": retry_at_future,
-        "seconds_until_retry": seconds_until_retry,
-        "worker_pid": report.crawl_process_pids.get(crawl_id),
-        "worker_state": crawl_worker_state,
-    }
-
-
-def render_snapshot(report, snapshot, crawl_id):
-    snapshot_run_started_at = snapshot.get("downloaded_at") or snapshot.get("created_at")
-    snapshot_process_started_at = snapshot.get("created_at")
-    # Get archive results only for displayed active snapshots. Large crawls can
-    # contain thousands of sealed snapshots, and prefetching all their results
-    # makes the progress endpoint compete with the runner.
-    snapshot_results = [
-        ar
-        for ar in report.archiveresults_by_snapshot.get(str(snapshot["id"]), [])
-        if archiveresult_matches_current_run(ar, snapshot_run_started_at)
-    ]
-    if snapshot["status"] == Snapshot.StatusChoices.QUEUED:
-        snapshot_results = []
-
-    plugin_progress_values: list[int] = []
-    all_plugins: list[dict[str, object]] = []
-    seen_plugin_keys: set[str] = set()
-    snapshot_title = (
-        str(snapshot["title"] or "")
-        if snapshot["status"] == Snapshot.StatusChoices.QUEUED
-        else Snapshot._normalize_title_candidate(snapshot["title"], snapshot_url=snapshot["url"])
-    )
-    snapshot_favicon_url = ""
-    snapshot_preview_url = ""
-    snapshot_preview_link = ""
-    snapshot_screencast_url = ""
-    snapshot_screencast_link = ""
-    snapshot_fallback_urls: list[str] = []
-    result_by_plugin = {result.plugin: result for result in snapshot_results}
-    title_result = result_by_plugin.get("title")
-    if not snapshot_title and title_result is not None and title_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
-        snapshot_title = Snapshot._normalize_title_candidate(title_result.output_str, snapshot_url=snapshot["url"])
-    favicon_result = result_by_plugin.get("favicon")
-    if favicon_result is not None and favicon_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
-        favicon_path = (
-            favicon_result.embed_path_db(
-                favicon_result.output_files if isinstance(favicon_result.output_files, dict) else {},
-                check_filesystem=False,
-            )
-            or "favicon/favicon.ico"
-        )
-        snapshot_favicon_url = snapshot_output_url(report, snapshot, favicon_path)
-    screenshot_result = result_by_plugin.get("screenshot")
-    if screenshot_result is not None and screenshot_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
-        snapshot_preview_link = snapshot_view_url(report, snapshot)
-        screenshot_path = (
-            screenshot_result.embed_path_db(
-                screenshot_result.output_files if isinstance(screenshot_result.output_files, dict) else {},
-                check_filesystem=False,
-            )
-            or "screenshot/screenshot.png"
-        )
-        snapshot_preview_url = snapshot_output_url(report, snapshot, screenshot_path)
-        snapshot_preview_link = snapshot_view_url(report, snapshot, screenshot_path)
-        if snapshot_favicon_url:
-            snapshot_fallback_urls.append(snapshot_favicon_url)
-    elif snapshot_favicon_url:
-        snapshot_preview_url = snapshot_favicon_url
-
-    if snapshot["status"] == Snapshot.StatusChoices.STARTED:
-        snapshot_screencast_url = screencast_frame_url(report, crawl_id, report.active_crawl_objects[crawl_id].output_dir)
-        snapshot_screencast_link = snapshot_view_url(report, snapshot) if snapshot_screencast_url else ""
-
-    def plugin_sort_key(ar):
-        status_order = {
-            ArchiveResult.StatusChoices.STARTED: 0,
-            ArchiveResult.StatusChoices.QUEUED: 1,
-            ArchiveResult.StatusChoices.SUCCEEDED: 2,
-            ArchiveResult.StatusChoices.NORESULTS: 3,
-            ArchiveResult.StatusChoices.FAILED: 4,
-        }
-        return (status_order.get(ar.status, 5), ar.plugin, ar.hook_name or "")
-
-    for ar in sorted(snapshot_results, key=plugin_sort_key):
-        status = ar.status
-        process = ar.process_record
-        progress_value = 0
-        if status in (
-            ArchiveResult.StatusChoices.SUCCEEDED,
-            ArchiveResult.StatusChoices.FAILED,
-            ArchiveResult.StatusChoices.SKIPPED,
-            ArchiveResult.StatusChoices.NORESULTS,
-        ):
-            progress_value = 100
-        elif status == ArchiveResult.StatusChoices.STARTED:
-            started_at = ar.start_ts or (process.started_at if process else None)
-            timeout = process.timeout if process else 120
-            if started_at and timeout:
-                elapsed = max(0.0, (report.now - started_at).total_seconds())
-                progress_value = int(min(99, max(1, (elapsed / float(timeout)) * 100)))
-            else:
-                progress_value = 1
-        else:
-            progress_value = 0
-
-        plugin_progress_values.append(progress_value)
-        plugin, label, phase, hook_name = hook_details(ar.hook_name or ar.plugin, plugin=ar.plugin)
-
-        plugin_payload = {
-            "id": str(ar.id),
-            "plugin": ar.plugin,
-            "label": label,
-            "hook_name": hook_name,
-            "phase": phase,
-            "status": status,
-            "process_id": str(process.id) if process else None,
-            "admin_url": f"/admin/core/archiveresult/{ar.id}/change/",
-        }
-        output_path = ar.embed_path_db(ar.output_files if isinstance(ar.output_files, dict) else {}, check_filesystem=False)
-        if output_path:
-            plugin_payload["output_path"] = output_path
-            plugin_payload["output_url"] = snapshot_view_url(report, snapshot, output_path)
-        if status == ArchiveResult.StatusChoices.STARTED and process:
-            plugin_payload["pid"] = process.pid
-        if status == ArchiveResult.StatusChoices.STARTED:
-            plugin_payload["progress"] = progress_value
-            plugin_payload["timeout"] = process.timeout if process else 120
-        plugin_payload["source"] = "archiveresult"
-        all_plugins.append(plugin_payload)
-        seen_plugin_keys.add(str(process.id) if process else f"{ar.plugin}:{hook_name}")
-
-    for proc_payload, proc_started_at in report.process_records_by_snapshot.get(str(snapshot["id"]), []):
-        if not is_current_run_timestamp(proc_started_at, snapshot_process_started_at):
-            continue
-        proc_key = str(proc_payload.get("process_id") or f"{proc_payload.get('plugin')}:{proc_payload.get('hook_name')}")
-        if proc_key in seen_plugin_keys:
-            continue
-        seen_plugin_keys.add(proc_key)
-        all_plugins.append(proc_payload)
-
-        proc_status = proc_payload.get("status")
-        if proc_status in ("succeeded", "failed", "skipped"):
-            plugin_progress_values.append(100)
-        elif proc_status == "started":
-            plugin_progress_values.append(1)
-        else:
-            plugin_progress_values.append(0)
-
-    total_plugins = len(all_plugins)
-    completed_plugins = sum(1 for item in all_plugins if item.get("status") == "succeeded")
-    failed_plugins = sum(1 for item in all_plugins if item.get("status") == "failed")
-    pending_plugins = sum(1 for item in all_plugins if item.get("status") == "queued")
-
-    snapshot_progress = int(sum(plugin_progress_values) / len(plugin_progress_values)) if plugin_progress_values else 0
-    worker_state = "running" if report.snapshot_process_pids.get(str(snapshot["id"])) else "waiting"
-    if (
-        snapshot["status"] == Snapshot.StatusChoices.STARTED
-        and worker_state == "waiting"
-        and not all_plugins
-        and snapshot["modified_at"]
-        and (report.now - snapshot["modified_at"]).total_seconds() > 30
-    ):
-        worker_state = "waiting" if report.orchestrator_running else "crashed"
-
-    if snapshot["status"] == Snapshot.StatusChoices.QUEUED and not report.snapshot_process_pids.get(str(snapshot["id"])):
-        compact_snapshot = [
-            str(snapshot["id"]),
-            snapshot_display_url(snapshot["url"]),
-        ]
-        if snapshot_title:
-            compact_snapshot.append(snapshot_title)
-        return compact_snapshot
-
-    snapshot_payload = {
-        "id": str(snapshot["id"]),
-        "url": snapshot_display_url(snapshot["url"]),
-        "title": snapshot_title,
-        "status": snapshot["status"],
-        "worker_state": worker_state,
-    }
-    if snapshot["status"] != Snapshot.StatusChoices.QUEUED or all_plugins or report.snapshot_process_pids.get(str(snapshot["id"])):
-        snapshot_payload.update(
-            {
-                "view_url": snapshot_view_url(report, snapshot),
-                "started": (snapshot["downloaded_at"] or snapshot["created_at"]).isoformat()
-                if (snapshot["downloaded_at"] or snapshot["created_at"])
-                else None,
-                "progress": snapshot_progress,
-                "total_plugins": total_plugins,
-                "completed_plugins": completed_plugins,
-                "failed_plugins": failed_plugins,
-                "pending_plugins": pending_plugins,
-                "all_plugins": all_plugins,
-            },
-        )
-        if snapshot_favicon_url:
-            snapshot_payload["favicon_url"] = snapshot_favicon_url
-        if snapshot_preview_url:
-            snapshot_payload["preview_url"] = snapshot_preview_url
-            snapshot_payload["preview_link"] = snapshot_preview_link
-        if snapshot_screencast_url:
-            snapshot_payload["screencast_url"] = snapshot_screencast_url
-            snapshot_payload["screencast_link"] = snapshot_screencast_link
-        if snapshot_fallback_urls:
-            snapshot_payload["preview_fallbacks"] = snapshot_fallback_urls
-        if report.snapshot_process_pids.get(str(snapshot["id"])):
-            snapshot_payload["worker_pid"] = report.snapshot_process_pids[str(snapshot["id"])]
-
-    return snapshot_payload
-
-
-def is_current_run_timestamp(event_ts, run_started_at) -> bool:
-    if run_started_at is None:
-        return True
-    if event_ts is None:
-        return False
-    return event_ts >= run_started_at
-
-
-def archiveresult_matches_current_run(ar, run_started_at) -> bool:
-    if run_started_at is None:
-        return True
-    if ar.status in (
-        ArchiveResult.StatusChoices.QUEUED,
-        ArchiveResult.StatusChoices.STARTED,
-        ArchiveResult.StatusChoices.BACKOFF,
-    ):
-        return True
-    event_ts = ar.end_ts or ar.start_ts or ar.modified_at or ar.created_at
-    return is_current_run_timestamp(event_ts, run_started_at)
-
-
-def hook_details(hook_name: str, plugin: str = "setup") -> tuple[str, str, str, str]:
-    normalized_hook_name = Path(hook_name).name if hook_name else ""
-    if not normalized_hook_name:
-        return (plugin, plugin, "unknown", "")
-
-    phase = "unknown"
-    if normalized_hook_name == "InstallEvent":
-        phase = "install"
-    elif normalized_hook_name.startswith("on_CrawlSetup__"):
-        phase = "crawl"
-    elif normalized_hook_name.startswith("on_Snapshot__"):
-        phase = "snapshot"
-
-    label = normalized_hook_name
-    if "__" in normalized_hook_name:
-        label = normalized_hook_name.split("__", 1)[1]
-    label = label.rsplit(".", 1)[0]
-    if len(label) > 3 and label[:2].isdigit() and label[2] == "_":
-        label = label[3:]
-    label = label.replace("_", " ").strip() or plugin
-
-    return (plugin, label, phase, normalized_hook_name)
-
-
-def process_label(cmd: list[str] | None) -> tuple[str, str, str, str]:
-    hook_path = ""
-    if isinstance(cmd, list) and cmd:
-        hook_path = next(
-            (arg for arg in cmd if isinstance(arg, str) and Path(arg).name.startswith("on_") and "__" in Path(arg).name),
-            "",
-        )
-        if not hook_path and isinstance(cmd[0], str):
-            hook_path = cmd[0]
-
-    if not hook_path:
-        return ("", "setup", "unknown", "")
-
-    return hook_details(Path(hook_path).name, plugin=Path(hook_path).parent.name or "setup")
-
-
-def snapshot_output_url(report, snapshot, output_path: str) -> str:
-    return build_snapshot_url(str(snapshot["id"]), output_path, request=report.request, config=report.request_config)
-
-
-def snapshot_archive_path(report, snapshot) -> str:
-    if snapshot["fs_version"] in ("0.7.0", "0.8.0"):
-        return f"{CONSTANTS.ARCHIVE_DIR_NAME}/{snapshot['timestamp']}"
-    crawl = report.crawls_by_id.get(str(snapshot["crawl_id"]))
-    username = "web"
-    if crawl is not None and crawl["created_by_id"]:
-        username = crawl["created_by__username"]
-    if username == "system":
-        username = "web"
-    date_base = snapshot["bookmarked_at"] or snapshot["created_at"]
-    date_str = date_base.strftime("%Y%m%d") if date_base else "unknown"
-    domain = Snapshot.extract_domain_from_url(snapshot["url"])
-    return f"{username}/{date_str}/{domain}/{snapshot['id']}"
-
-
-def snapshot_view_url(report, snapshot, output_path: str = "") -> str:
-    anchor = f"#{output_path}" if output_path else ""
-    return build_web_url(
-        f"/{snapshot_archive_path(report, snapshot)}/index.html{anchor}",
-        request=report.request,
-        config=report.request_config,
-    )
-
-
-def snapshot_display_url(url: str) -> str:
-    url = str(url or "")
-    return url if len(url) <= 96 else f"{url[:93]}..."
-
-
-def screencast_frame_url(report, crawl_id: str, crawl_dir: Path) -> str:
-    frame_path = crawl_dir / "chrome_screencast" / "latest.jpg"
-    try:
-        frame_stat = frame_path.stat()
-    except OSError:
-        return ""
-    if frame_stat.st_size <= 0:
-        return ""
-    if report.now.timestamp() - frame_stat.st_mtime > 15:
-        return ""
-    rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}"
-    return f"{report.api_base}{rel}" if report.api_base else rel

archivebox/progressmonitor/report.py

@@ -1,426 +0,0 @@
-"""Assemble one live-progress report from scoped, bounded database queries."""
-
-from functools import lru_cache
-from typing import Any
-from pathlib import Path
-
-from abx_dl.events import PROCESS_EXIT_SKIPPED
-from django.db.models import CharField, Count, Q, Sum
-from django.db.models.functions import Cast
-from django.utils import timezone
-
-from archivebox.core.routes_util import get_api_base_url
-from archivebox.plugins.discovery import discover_plugin_configs
-
-
-from archivebox.core.models import ArchiveResult, Snapshot
-from archivebox.crawls.models import Crawl
-from archivebox.machine.models import Machine, Process
-
-from .presentation import render_crawl, process_label
-from datetime import timedelta
-
-
-@lru_cache(maxsize=1)
-def _live_progress_plugin_names() -> tuple[frozenset[str], frozenset[str]]:
-    plugin_configs = discover_plugin_configs()
-    indexing_plugin_names = frozenset(
-        plugin_name
-        for plugin_name, plugin_config in plugin_configs.items()
-        if {"search", "flush"}.issubset(plugin_config.get("commands", {}))
-    )
-    download_plugin_names = frozenset(
-        plugin_name
-        for plugin_name, plugin_config in plugin_configs.items()
-        if plugin_config.get("output_mimetypes") and plugin_name not in indexing_plugin_names
-    )
-    return download_plugin_names, indexing_plugin_names
-
-
-class ProgressReport:
-    """Per-request query results shared by status, process, and preview rendering."""
-
-    def __init__(self, request, *, scoped_snapshot, crawl_id_filter, is_admin):
-        self.request = request
-        self.scoped_snapshot = scoped_snapshot
-        self.crawl_id_filter = crawl_id_filter
-        self.is_admin = is_admin
-        self.request_config = self.request.archivebox_config
-        self.now = timezone.now()
-        self.crawl_scope = Crawl.objects.all()
-        self.snapshot_scope = Snapshot.objects.all()
-        self.archiveresult_scope = ArchiveResult.objects.all()
-        if self.is_admin and not self.request.user.is_superuser:
-            self.crawl_scope = self.crawl_scope.filter(created_by=self.request.user)
-            self.snapshot_scope = self.snapshot_scope.filter(crawl__created_by=self.request.user)
-            self.archiveresult_scope = self.archiveresult_scope.filter(snapshot__crawl__created_by=self.request.user)
-        if self.scoped_snapshot is not None:
-            self.snapshot_scope = Snapshot.objects.filter(id=self.scoped_snapshot.id)
-            self.crawl_scope = Crawl.objects.filter(id=self.scoped_snapshot.crawl_id)
-            self.archiveresult_scope = ArchiveResult.objects.filter(snapshot_id=self.scoped_snapshot.id)
-        elif self.crawl_id_filter:
-            self.snapshot_scope = self.snapshot_scope.filter(crawl_id=self.crawl_id_filter)
-            self.crawl_scope = self.crawl_scope.filter(id=self.crawl_id_filter)
-            self.archiveresult_scope = self.archiveresult_scope.filter(snapshot__crawl_id=self.crawl_id_filter)
-
-        self.api_base = get_api_base_url(request=self.request, config=self.request_config) if self.scoped_snapshot is not None else ""
-
-    def load_orchestrator(self):
-        self.machine_id = Machine.current().id
-        orchestrator_proc = (
-            Process.objects.filter(
-                machine_id=self.machine_id,
-                process_type=Process.TypeChoices.ORCHESTRATOR,
-                status=Process.StatusChoices.RUNNING,
-            )
-            .only("id", "pid", "started_at", "machine_id", "process_type", "status")
-            .order_by("-started_at")
-            .first()
-            if self.machine_id is not None
-            else None
-        )
-        runner_worker = None
-        orchestrator_proc_running = bool(orchestrator_proc and orchestrator_proc.is_running)
-        if not orchestrator_proc_running:
-            try:
-                from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
-
-                supervisor = get_existing_supervisord_process(quiet=True)
-                runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
-            except (OSError, RuntimeError, TimeoutError):
-                runner_worker = None
-
-        runner_worker_running = bool(runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"))
-        runner_worker_pid = runner_worker.get("pid") if runner_worker else None
-        self.orchestrator_running = orchestrator_proc_running or runner_worker_running
-        self.orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else runner_worker_pid
-
-    def load_counts(self):
-        downloads, indexing = _live_progress_plugin_names()
-        download_scope = self.archiveresult_scope.filter(
-            plugin__in=downloads,
-            snapshot__status__in=Snapshot.RUNNABLE_STATES,
-            snapshot__crawl__status__in=Crawl.RUNNABLE_STATES,
-        )
-        self.counts = {}
-        for label, model, scope in (
-            ("crawls", Crawl, self.crawl_scope),
-            ("snapshots", Snapshot, self.snapshot_scope),
-            ("archiveresults", ArchiveResult, self.archiveresult_scope),
-            ("downloads", ArchiveResult, download_scope),
-            ("indexing", ArchiveResult, self.archiveresult_scope.filter(plugin__in=indexing)),
-        ):
-            counts = model.status_counts(scope, ("queued", "started"))
-            self.counts[f"{label}_queued"] = counts.get("queued", 0)
-            self.counts[f"{label}_active"] = counts.get("started", 0)
-        self.counts["crawls_recent"] = self.crawl_scope.filter(created_at__gte=self.now - timedelta(days=1)).count()
-
-    def load_crawls(self):
-        # Build hierarchical active crawls with nested snapshots and archive results
-        max_active_crawls = 10
-        max_queued_crawls = 10
-        self.max_started_snapshots_per_crawl = 50
-        self.max_queued_snapshots_per_crawl = 50
-
-        active_crawl_fields = (
-            "id",
-            "created_at",
-            "created_by_id",
-            "modified_at",
-            "urls",
-            "config",
-            "max_depth",
-            "tags_str",
-            "persona_id",
-            "status",
-            "retry_at",
-            "label",
-            "created_by__id",
-            "created_by__username",
-        )
-        started_crawls = list(
-            self.crawl_scope.filter(status=Crawl.StatusChoices.STARTED)
-            .values(*active_crawl_fields)
-            .order_by("-modified_at")[:max_active_crawls],
-        )
-        paused_crawls = list(
-            self.crawl_scope.filter(
-                Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=self.now - timedelta(hours=12))
-                | Q(
-                    status=Crawl.StatusChoices.PAUSED,
-                    snapshot_set__status__in=Snapshot.RUNNABLE_STATES,
-                    snapshot_set__retry_at__lte=self.now,
-                )
-                | Q(
-                    status=Crawl.StatusChoices.PAUSED,
-                    snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
-                ),
-            )
-            .values(*active_crawl_fields)
-            .distinct()
-            .order_by("-modified_at")[:max_active_crawls],
-        )
-        queued_crawls = list(
-            self.crawl_scope.filter(status=Crawl.StatusChoices.QUEUED)
-            .values(*active_crawl_fields)
-            .order_by("-modified_at")[:max_queued_crawls],
-        )
-        self.queued_crawls_hidden = max(self.counts["crawls_queued"] - len(queued_crawls), 0)
-        self.active_crawls_list = started_crawls + paused_crawls + queued_crawls
-        for crawl in self.active_crawls_list:
-            crawl["id"] = str(crawl["id"])
-            if crawl["persona_id"]:
-                crawl["persona_id"] = str(crawl["persona_id"])
-        self.persona_details_by_id: dict[str, dict[str, str]] = {}
-        self.persona_details_by_name: dict[str, dict[str, str]] = {}
-        persona_ids = {crawl["persona_id"] for crawl in self.active_crawls_list if crawl["persona_id"]}
-        persona_names = {"Default"} if any(not crawl["persona_id"] for crawl in self.active_crawls_list) else set()
-        if persona_ids or persona_names:
-            from archivebox.personas.models import Persona
-
-            for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name", "config"):
-                persona_details = {
-                    "name": persona.name,
-                    "admin_url": f"/admin/personas/persona/{persona.pk}/change/",
-                }
-                self.persona_details_by_id[str(persona.id)] = persona_details
-                self.persona_details_by_name[persona.name] = persona_details
-        self.active_crawl_ids = [crawl["id"] for crawl in self.active_crawls_list]
-        self.active_crawl_objects = {}
-        if self.active_crawl_ids:
-            for crawl_obj in Crawl.objects.filter(id__in=self.active_crawl_ids).select_related("created_by", "persona"):
-                crawl_obj._runtime_config = self.request_config
-                self.active_crawl_objects[str(crawl_obj.id)] = crawl_obj
-        self.snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in self.active_crawl_ids}
-        self.cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in self.active_crawl_ids}
-        self.crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in self.active_crawl_ids}
-        self.queued_snapshot_overflow_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in self.active_crawl_ids}
-        self.active_snapshot_scope = self.snapshot_scope.filter(crawl_id__in=self.active_crawl_ids)
-        if self.active_crawl_ids:
-            for row in self.active_snapshot_scope.values("crawl_id", "status").annotate(count=Count("id")):
-                self.snapshot_counts_by_crawl.setdefault(str(row["crawl_id"]), {})[row["status"]] = row["count"]
-
-            for row in (
-                self.active_snapshot_scope.filter(status=Snapshot.StatusChoices.SEALED, downloaded_at__isnull=True)
-                .values("crawl_id")
-                .annotate(count=Count("id"))
-            ):
-                self.cancelled_snapshot_counts_by_crawl[str(row["crawl_id"])] = row["count"]
-
-            for row in (
-                self.active_snapshot_scope.filter(
-                    status=Snapshot.StatusChoices.SEALED,
-                )
-                .values("crawl_id")
-                .annotate(size=Sum("output_size"))
-            ):
-                self.crawl_output_sizes_by_crawl[str(row["crawl_id"])] = int(row["size"] or 0)
-
-    def load_snapshots(self):
-        self.crawl_process_pids: dict[str, int] = {}
-        self.snapshot_process_pids: dict[str, int] = {}
-        self.process_records_by_crawl: dict[str, list[tuple[dict[str, object], object | None]]] = {}
-        self.process_records_by_snapshot: dict[str, list[tuple[dict[str, object], object | None]]] = {}
-        self.seen_process_records: set[str] = set()
-        self.crawls_by_id = {str(crawl["id"]): crawl for crawl in self.active_crawls_list}
-        started_snapshot_fields = (
-            "id_str",
-            "created_at",
-            "modified_at",
-            "url",
-            "timestamp",
-            "bookmarked_at",
-            "crawl_id_str",
-            "title",
-            "downloaded_at",
-            "fs_version",
-            "status",
-        )
-        queued_snapshot_fields = (
-            "id_str",
-            "url",
-            "crawl_id_str",
-            "title",
-            "status",
-        )
-        snapshots = []
-        for crawl_id in self.active_crawl_ids:
-            crawl_snapshot_scope = self.active_snapshot_scope.filter(crawl_id=crawl_id)
-            snapshots.extend(
-                crawl_snapshot_scope.filter(status=Snapshot.StatusChoices.STARTED)
-                .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
-                .values(*started_snapshot_fields)
-                .order_by("-modified_at")[: self.max_started_snapshots_per_crawl],
-            )
-            queued_snapshots = list(
-                crawl_snapshot_scope.filter(status=Snapshot.StatusChoices.QUEUED)
-                .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
-                .values(
-                    *queued_snapshot_fields,
-                )
-                .order_by("modified_at")[: self.max_queued_snapshots_per_crawl],
-            )
-            self.queued_snapshot_overflow_by_crawl[str(crawl_id)] = max(
-                self.snapshot_counts_by_crawl.get(str(crawl_id), {}).get(Snapshot.StatusChoices.QUEUED, 0) - len(queued_snapshots),
-                0,
-            )
-            snapshots.extend(queued_snapshots)
-
-        for snapshot in snapshots:
-            # Process.pwd points at Snapshot.output_dir, which uses CompactUUID
-            # hex path components. Keep progress IDs compact too so process rows
-            # can be matched without carrying dashed/undashed variants.
-            snapshot["id"] = str(snapshot.pop("id_str")).replace("-", "")
-            snapshot["crawl_id"] = str(snapshot.pop("crawl_id_str")).replace("-", "")
-        self.snapshots_by_id = {str(snapshot["id"]): snapshot for snapshot in snapshots}
-        self.displayed_snapshots_by_crawl: dict[str, list[dict[str, Any]]] = {str(crawl_id): [] for crawl_id in self.active_crawl_ids}
-        for snapshot in snapshots:
-            crawl_snapshots = self.displayed_snapshots_by_crawl.setdefault(str(snapshot["crawl_id"]), [])
-            crawl_snapshots.append(snapshot)
-        displayed_snapshot_ids = [
-            snapshot["id"] for crawl_snapshots in self.displayed_snapshots_by_crawl.values() for snapshot in crawl_snapshots
-        ]
-        detailed_snapshot_ids = [snapshot["id"] for snapshot in snapshots if snapshot["status"] != Snapshot.StatusChoices.QUEUED]
-        process_value_fields = ("id", "process_type", "status", "pwd", "cmd", "pid", "exit_code", "started_at", "ended_at", "modified_at")
-        if self.active_crawl_ids or displayed_snapshot_ids:
-            process_scope = Process.objects.filter(
-                machine_id=self.machine_id,
-                process_type__in=[
-                    Process.TypeChoices.HOOK,
-                    Process.TypeChoices.BINARY,
-                ],
-            )
-            self.running_processes = process_scope.filter(status=Process.StatusChoices.RUNNING).values(*process_value_fields)
-            self.recent_processes = (
-                process_scope.filter(modified_at__gte=self.now - timedelta(minutes=10))
-                .values(*process_value_fields)
-                .order_by("-modified_at")
-            )
-        else:
-            self.running_processes = Process.objects.none().values(*process_value_fields)
-            self.recent_processes = Process.objects.none().values(*process_value_fields)
-
-        self.archiveresults_by_snapshot: dict[str, list[ArchiveResult]] = {str(snapshot_id): [] for snapshot_id in detailed_snapshot_ids}
-        if detailed_snapshot_ids:
-            displayed_archiveresults = (
-                self.archiveresult_scope.filter(snapshot_id__in=detailed_snapshot_ids)
-                .select_related("process")
-                .only(
-                    "id",
-                    "snapshot_id",
-                    "plugin",
-                    "hook_name",
-                    "status",
-                    "output_str",
-                    "output_files",
-                    "output_size",
-                    "start_ts",
-                    "end_ts",
-                    "created_at",
-                    "modified_at",
-                    "process_id",
-                    "process__id",
-                    "process__pid",
-                    "process__started_at",
-                    "process__timeout",
-                )
-                .order_by("snapshot_id", "start_ts", "created_at")
-            )
-            for archiveresult in displayed_archiveresults:
-                self.archiveresults_by_snapshot.setdefault(str(archiveresult.snapshot_id), []).append(archiveresult)
-
-    def load_processes(self):
-        self.running_worker_ids: set[str] = set()
-        for proc in self.running_processes:
-            if not proc["pwd"]:
-                continue
-            scope = self.process_scope_ids(Path(proc["pwd"]))
-            if scope is None:
-                continue
-            crawl_id, snapshot_id = scope
-            self.running_worker_ids.add(str(proc["id"]))
-            _plugin, _label, phase, _hook_name = process_label(proc["cmd"])
-            if crawl_id and proc["pid"]:
-                self.crawl_process_pids.setdefault(crawl_id, proc["pid"])
-            if phase == "snapshot" and snapshot_id and proc["pid"]:
-                self.snapshot_process_pids.setdefault(snapshot_id, proc["pid"])
-
-        for proc in self.recent_processes:
-            if not proc["pwd"]:
-                continue
-            scope = self.process_scope_ids(Path(proc["pwd"]))
-            if scope is None:
-                continue
-            crawl_id, snapshot_id = scope
-
-            plugin, label, phase, hook_name = process_label(proc["cmd"])
-
-            record_scope = str(snapshot_id) if phase == "snapshot" and snapshot_id else str(crawl_id)
-            proc_key = f"{record_scope}:{plugin}:{label}:{proc['status']}:{proc['exit_code']}"
-            if proc_key in self.seen_process_records:
-                continue
-            self.seen_process_records.add(proc_key)
-
-            status = (
-                "started"
-                if proc["status"] == Process.StatusChoices.RUNNING
-                else (
-                    "skipped"
-                    if proc["exit_code"] == PROCESS_EXIT_SKIPPED or (phase == "binary" and proc["exit_code"] not in (None, 0))
-                    else ("failed" if proc["exit_code"] not in (None, 0) else "succeeded")
-                )
-            )
-            payload: dict[str, object] = {
-                "id": str(proc["id"]),
-                "plugin": plugin,
-                "label": label,
-                "hook_name": hook_name,
-                "status": status,
-                "phase": phase,
-                "source": "process",
-                "process_id": str(proc["id"]),
-            }
-            if status == "started" and proc["pid"]:
-                payload["pid"] = proc["pid"]
-            proc_started_at = proc["started_at"] or proc["modified_at"]
-            proc_run_at = (
-                proc_started_at
-                if proc["status"] == Process.StatusChoices.RUNNING
-                else (proc["ended_at"] or proc["modified_at"] or proc_started_at)
-            )
-            if phase == "snapshot" and snapshot_id:
-                self.process_records_by_snapshot.setdefault(snapshot_id, []).append((payload, proc_run_at))
-            elif crawl_id:
-                self.process_records_by_crawl.setdefault(crawl_id, []).append((payload, proc_run_at))
-
-    def payload(self):
-        self.load_orchestrator()
-        self.load_counts()
-        self.load_crawls()
-        self.load_snapshots()
-        self.load_processes()
-        return {
-            "is_admin": self.is_admin,
-            "scope": {
-                "snapshot_id": str(self.scoped_snapshot.id) if self.scoped_snapshot is not None else "",
-                "crawl_id": self.crawl_id_filter,
-            },
-            "orchestrator_running": self.orchestrator_running,
-            "orchestrator_pid": self.orchestrator_pid,
-            "total_workers": len(self.running_worker_ids),
-            **self.counts,
-            "active_crawls": [render_crawl(self, crawl) for crawl in self.active_crawls_list],
-            "queued_crawls_hidden": self.queued_crawls_hidden,
-            "server_time": timezone.now().isoformat(),
-        }
-
-    def process_scope_ids(self, proc_pwd: Path) -> tuple[str, str] | None:
-        """Match a process directory to the displayed snapshot or crawl IDs."""
-        parts = [part.replace("-", "") for part in reversed(proc_pwd.parts)]
-        for part in parts:
-            if snapshot := self.snapshots_by_id.get(part):
-                return str(snapshot["crawl_id"]), str(snapshot["id"])
-        for part in parts:
-            if crawl := self.crawls_by_id.get(part):
-                return str(crawl["id"]), ""
-        return None

archivebox/progressmonitor/static/progressmonitor/monitor.css

@@ -1,984 +0,0 @@
-/* Progress Monitor Container */
-#progress-monitor {
-    background: linear-gradient(135deg, #0d1117 0%, #161b22 100%);
-    color: #c9d1d9;
-    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Noto Sans', Helvetica, Arial, sans-serif;
-    font-size: 12px;
-    border-bottom: 1px solid #30363d;
-    position: relative;
-    z-index: 100;
-}
-#progress-monitor.hidden {
-    display: none;
-}
-#progress-monitor .tree-container {
-    max-height: 350px;
-    overflow-y: auto;
-}
-#progress-monitor .progress-content {
-    display: flex;
-    align-items: flex-start;
-    gap: 12px;
-    padding: 12px 16px;
-}
-
-/* Header Bar */
-#progress-monitor .header-bar {
-    display: flex;
-    justify-content: space-between;
-    align-items: center;
-    padding: 8px 16px;
-    background: rgba(0,0,0,0.2);
-    border-bottom: 1px solid #30363d;
-    position: sticky;
-    top: 0;
-    z-index: 10;
-}
-#progress-monitor.collapsed .header-bar {
-    cursor: pointer;
-}
-#progress-monitor .header-left {
-    display: flex;
-    align-items: center;
-    gap: 16px;
-    min-width: 0;
-}
-#progress-monitor .header-right {
-    display: flex;
-    align-items: center;
-    gap: 12px;
-    flex-shrink: 0;
-}
-
-/* Orchestrator Status */
-#progress-monitor .orchestrator-status {
-    display: flex;
-    align-items: center;
-    gap: 6px;
-}
-#progress-monitor .status-dot {
-    width: 8px;
-    height: 8px;
-    border-radius: 50%;
-    flex-shrink: 0;
-}
-#progress-monitor .status-dot.running {
-    background: #238636;
-    box-shadow: 0 0 8px #238636;
-    animation: pulse 2s infinite;
-}
-#progress-monitor .status-dot.idle {
-    background: #7ee787;
-    box-shadow: 0 0 4px #7ee787;
-    animation: idle-pulse 5s infinite;
-}
-#progress-monitor .status-dot.stopped {
-    background: #d29922;
-    box-shadow: 0 0 4px #d29922;
-}
-#progress-monitor .status-dot.error {
-    background: #f85149;
-    box-shadow: 0 0 8px #f85149;
-}
-#progress-monitor .status-dot.flash {
-    animation: flash 0.3s ease-out;
-}
-@keyframes pulse {
-    0%, 100% { opacity: 1; box-shadow: 0 0 8px #238636; }
-    50% { opacity: 0.6; box-shadow: 0 0 4px #238636; }
-}
-@keyframes idle-pulse {
-    0%, 100% { opacity: 1; box-shadow: 0 0 4px #7ee787; }
-    50% { opacity: 0.65; box-shadow: 0 0 2px #7ee787; }
-}
-@keyframes flash {
-    0% { transform: scale(1.5); }
-    100% { transform: scale(1); }
-}
-
-/* Stats */
-#progress-monitor .stats {
-    display: flex;
-    gap: 16px;
-}
-#progress-monitor .stat {
-    display: flex;
-    align-items: center;
-    gap: 4px;
-}
-#progress-monitor .stat-label {
-    color: #8b949e;
-    font-size: 10px;
-    text-transform: uppercase;
-    letter-spacing: 0.5px;
-}
-#progress-monitor .stat-value {
-    font-weight: 600;
-    font-variant-numeric: tabular-nums;
-}
-#progress-monitor .stat-value.compact {
-    font-size: 11px;
-    white-space: nowrap;
-}
-#progress-monitor .stat-value.success { color: #3fb950; }
-#progress-monitor .stat-value.error { color: #f85149; }
-#progress-monitor .stat-value.warning { color: #d29922; }
-#progress-monitor .stat-value.info { color: #58a6ff; }
-
-/* Toggle Button */
-#progress-monitor .toggle-btn {
-    background: transparent;
-    border: 1px solid #30363d;
-    color: #8b949e;
-    cursor: pointer;
-    padding: 4px 8px;
-    border-radius: 6px;
-    font-size: 11px;
-    transition: all 0.2s;
-}
-#progress-monitor .toggle-btn:hover {
-    background: #21262d;
-    color: #c9d1d9;
-    border-color: #8b949e;
-}
-#progress-monitor .crawl-action-btn,
-#progress-monitor .cancel-item-btn {
-    background: transparent;
-    border: 1px solid #30363d;
-    cursor: pointer;
-    padding: 2px 6px;
-    border-radius: 6px;
-    font-size: 11px;
-    line-height: 1;
-    transition: all 0.2s;
-    flex-shrink: 0;
-}
-#progress-monitor .pause-item-btn {
-    color: #d29922;
-}
-#progress-monitor .pause-item-btn:hover {
-    background: rgba(210, 153, 34, 0.12);
-    border-color: #d29922;
-    color: #f0b72f;
-}
-#progress-monitor .cancel-item-btn {
-    color: #f85149;
-}
-#progress-monitor .cancel-item-btn:hover {
-    background: rgba(248, 81, 73, 0.12);
-    border-color: #f85149;
-    color: #ff7b72;
-}
-#progress-monitor .crawl-action-btn.is-busy,
-#progress-monitor .cancel-item-btn.is-busy {
-    opacity: 0.6;
-    cursor: wait;
-    border-color: #6e7681;
-    color: #6e7681;
-}
-#progress-monitor .progress-overflow-note {
-    padding: 8px 12px;
-    color: #8b949e;
-    font-size: 11px;
-    text-align: center;
-    background: rgba(33, 38, 45, 0.55);
-    border-top: 1px solid #30363d;
-}
-
-/* Tree Container */
-#progress-monitor .tree-container {
-    flex: 1 1 auto;
-    min-width: 0;
-    padding: 0;
-}
-#progress-monitor.collapsed .progress-content {
-    display: none;
-}
-/* Hide admin-only controls when viewer is not staff, or when the monitor is
-   embedded on a non-admin host (snap-* subdomain) where cross-origin POSTs
-   to /api would violate the subdomain isolation model. */
-#progress-monitor.is-guest .crawl-action-btn,
-#progress-monitor.is-guest .cancel-item-btn,
-#progress-monitor.is-guest .pause-item-btn,
-#progress-monitor[data-progress-scope="snapshot"] .crawl-action-btn,
-#progress-monitor[data-progress-scope="snapshot"] .cancel-item-btn,
-#progress-monitor[data-progress-scope="snapshot"] .pause-item-btn {
-    display: none !important;
-}
-
-/* Chrome Screencast */
-#progress-monitor .screencast-panel {
-    display: none;
-    flex: 0 0 336px;
-    width: 336px;
-    position: sticky;
-    top: 49px;
-    border: 1px solid rgba(88, 166, 255, 0.5);
-    border-radius: 8px;
-    overflow: hidden;
-    background: #0d1117;
-    box-shadow: 0 12px 34px rgba(1, 4, 9, 0.4), 0 0 0 1px rgba(88, 166, 255, 0.12);
-}
-#progress-monitor .screencast-panel.visible {
-    display: block;
-}
-#progress-monitor .screencast-frame {
-    display: block;
-    width: 100%;
-    height: 310px;
-    background: #010409;
-    color: #6e7681;
-    text-decoration: none;
-    overflow: hidden;
-}
-#progress-monitor .screencast-frame img {
-    display: block;
-    width: 120%;
-    height: 100%;
-    margin-left: -10%;
-    object-fit: cover;
-    object-position: top center;
-    background: #ffffff;
-    opacity: 1 !important;
-}
-#progress-monitor .screencast-placeholder {
-    display: flex;
-    width: 100%;
-    height: 100%;
-    align-items: center;
-    justify-content: center;
-    flex-direction: column;
-    gap: 6px;
-    text-align: center;
-    color: #8b949e;
-    background: #010409;
-}
-#progress-monitor .screencast-placeholder-icon {
-    color: #c9d1d9;
-    font-size: 32px;
-    font-weight: 700;
-    line-height: 1;
-}
-#progress-monitor .screencast-placeholder.launching .screencast-placeholder-icon {
-    color: #d29922;
-}
-#progress-monitor .screencast-placeholder-title {
-    color: #f0f6fc;
-    font-size: 13px;
-    font-weight: 700;
-    line-height: 1.25;
-}
-#progress-monitor .screencast-placeholder-subtitle {
-    max-width: 260px;
-    color: #8b949e;
-    font-size: 11px;
-    line-height: 1.35;
-}
-#progress-monitor .screencast-caption {
-    display: flex;
-    align-items: center;
-    gap: 8px;
-    padding: 8px 10px;
-    border-top: 1px solid #21262d;
-    background: rgba(13, 17, 23, 0.96);
-    color: inherit;
-    text-decoration: none;
-}
-#progress-monitor .screencast-dot {
-    width: 7px;
-    height: 7px;
-    border-radius: 50%;
-    flex: 0 0 7px;
-    background: #3fb950;
-    box-shadow: 0 0 8px rgba(63, 185, 80, 0.9);
-}
-#progress-monitor .screencast-dot.not-launched {
-    background: #6e7681;
-    box-shadow: none;
-}
-#progress-monitor .screencast-dot.launching {
-    background: #d29922;
-    box-shadow: 0 0 8px rgba(210, 153, 34, 0.8);
-}
-#progress-monitor .screencast-text {
-    min-width: 0;
-}
-#progress-monitor .screencast-title {
-    display: block;
-    color: #f0f6fc;
-    font-size: 12px;
-    font-weight: 600;
-    line-height: 1.25;
-    white-space: nowrap;
-    overflow: hidden;
-    text-overflow: ellipsis;
-}
-#progress-monitor .screencast-url {
-    display: block;
-    color: #8b949e;
-    font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
-    font-size: 10px;
-    line-height: 1.25;
-    white-space: nowrap;
-    overflow: hidden;
-    text-overflow: ellipsis;
-}
-
-/* Idle Message */
-#progress-monitor .idle-message {
-    color: #8b949e;
-    font-style: italic;
-    padding: 8px 0;
-    text-align: center;
-}
-
-/* Crawl Item */
-#progress-monitor .crawl-item {
-    background: #161b22;
-    border: 1px solid #30363d;
-    border-radius: 8px;
-    margin-bottom: 9px;
-    overflow: hidden;
-}
-#progress-monitor .crawl-header {
-    display: flex;
-    align-items: center;
-    gap: 10px;
-    padding: 8px 12px;
-    background: rgba(0,0,0,0.2);
-}
-#progress-monitor .crawl-header:hover {
-    background: rgba(88, 166, 255, 0.1);
-}
-#progress-monitor .crawl-header-link {
-    display: flex;
-    align-items: center;
-    gap: 12px;
-    flex: 1;
-    min-width: 0;
-    cursor: pointer;
-    text-decoration: none;
-    color: inherit;
-}
-#progress-monitor .crawl-header-link,
-#progress-monitor .crawl-header-link * {
-    text-decoration: none !important;
-}
-#progress-monitor a.crawl-header-link:visited {
-    color: inherit;
-}
-#progress-monitor a.crawl-header-link:link,
-#progress-monitor a.crawl-header-link:visited,
-#progress-monitor a.crawl-header-link:hover,
-#progress-monitor a.crawl-header-link:active {
-    text-decoration: none !important;
-}
-#progress-monitor .crawl-info {
-    flex: 1;
-    min-width: 0;
-}
-#progress-monitor .crawl-label {
-    font-weight: 600;
-    color: #f0f6fc;
-    white-space: nowrap;
-    overflow: hidden;
-    text-overflow: ellipsis;
-}
-#progress-monitor .crawl-badges {
-    display: flex;
-    flex-wrap: wrap;
-    gap: 4px;
-    margin-top: 5px;
-}
-#progress-monitor .crawl-badge {
-    display: inline-flex;
-    align-items: center;
-    gap: 4px;
-    max-width: 240px;
-    padding: 2px 7px;
-    border-radius: 999px;
-    font-size: 11px;
-    line-height: 1.35;
-    color: #c9d1d9;
-    background: rgba(110, 118, 129, 0.16);
-    border: 1px solid rgba(110, 118, 129, 0.22);
-    white-space: nowrap;
-    overflow: hidden;
-    text-overflow: ellipsis;
-}
-#progress-monitor a.crawl-badge {
-    cursor: pointer;
-    text-decoration: none !important;
-}
-#progress-monitor a.crawl-badge:hover {
-    border-color: currentColor;
-    filter: brightness(1.14);
-}
-#progress-monitor .crawl-badge strong {
-    color: #8b949e;
-    font-weight: 600;
-}
-#progress-monitor .crawl-title-link {
-    display: inline-block;
-    max-width: 100%;
-    color: inherit;
-    text-decoration: none !important;
-    white-space: nowrap;
-    overflow: hidden;
-    text-overflow: ellipsis;
-}
-#progress-monitor .crawl-title-link:hover {
-    color: #79c0ff;
-}
-#progress-monitor .crawl-badge.persona {
-    color: #d2a8ff;
-    background: rgba(163, 113, 247, 0.16);
-    border-color: rgba(163, 113, 247, 0.28);
-}
-#progress-monitor .crawl-badge.limit {
-    color: #79c0ff;
-    background: rgba(88, 166, 255, 0.13);
-    border-color: rgba(88, 166, 255, 0.24);
-}
-#progress-monitor .crawl-badge.size {
-    color: #ffa657;
-    background: rgba(240, 136, 62, 0.13);
-    border-color: rgba(240, 136, 62, 0.24);
-}
-#progress-monitor .crawl-badge.count {
-    color: #7ee787;
-    background: rgba(63, 185, 80, 0.13);
-    border-color: rgba(63, 185, 80, 0.24);
-}
-#progress-monitor .crawl-badge.tag {
-    color: #a5d6ff;
-    background: rgba(31, 111, 235, 0.16);
-    border-color: rgba(31, 111, 235, 0.28);
-}
-#progress-monitor .crawl-stats {
-    display: flex;
-    gap: 6px;
-    flex-wrap: wrap;
-    justify-content: flex-end;
-    font-size: 11px;
-}
-
-/* Progress Bar */
-#progress-monitor .progress-bar-container {
-    height: 4px;
-    background: #21262d;
-    border-radius: 2px;
-    overflow: hidden;
-    position: relative;
-}
-#progress-monitor .progress-bar {
-    height: 100%;
-    border-radius: 2px;
-    transition: width 0.5s ease-out;
-    position: relative;
-}
-#progress-monitor .progress-bar.crawl {
-    background: linear-gradient(90deg, #238636 0%, #3fb950 100%);
-}
-#progress-monitor .progress-bar.snapshot {
-    background: linear-gradient(90deg, #1f6feb 0%, #58a6ff 100%);
-}
-#progress-monitor .progress-bar.extractor {
-    background: linear-gradient(90deg, #8957e5 0%, #a371f7 100%);
-}
-#progress-monitor .progress-bar.indeterminate {
-    background: linear-gradient(90deg, transparent 0%, #58a6ff 50%, transparent 100%);
-    animation: indeterminate 1.5s infinite linear;
-    width: 30% !important;
-}
-@keyframes indeterminate {
-    0% { transform: translateX(-100%); }
-    100% { transform: translateX(400%); }
-}
-
-/* Crawl Body */
-#progress-monitor .crawl-body {
-    padding: 0 14px 14px;
-}
-#progress-monitor .crawl-progress {
-    padding: 10px 14px;
-    border-bottom: 1px solid #21262d;
-}
-
-/* Snapshot List */
-#progress-monitor .snapshot-list {
-    margin-top: 8px;
-}
-#progress-monitor .snapshot-item {
-    background: #0d1117;
-    border: 1px solid #21262d;
-    border-radius: 6px;
-    margin-bottom: 8px;
-    overflow: hidden;
-}
-#progress-monitor .snapshot-header {
-    display: flex;
-    align-items: center;
-    gap: 10px;
-    padding: 8px 12px;
-}
-#progress-monitor .snapshot-header:hover {
-    background: rgba(88, 166, 255, 0.05);
-}
-#progress-monitor .snapshot-header-link {
-    display: flex;
-    align-items: center;
-    gap: 10px;
-    flex: 1;
-    min-width: 0;
-    cursor: pointer;
-    text-decoration: none;
-    color: inherit;
-}
-#progress-monitor a.snapshot-header-link:visited {
-    color: inherit;
-}
-#progress-monitor .snapshot-icon {
-    font-size: 14px;
-    width: 18px;
-    text-align: center;
-    color: #58a6ff;
-}
-#progress-monitor .snapshot-preview {
-    display: flex;
-    align-items: center;
-    justify-content: center;
-    width: 84px;
-    height: 52px;
-    flex: 0 0 84px;
-    border-radius: 6px;
-    border: 1px solid #30363d;
-    background: #161b22;
-    color: #6e7681;
-    overflow: hidden;
-    text-decoration: none;
-}
-#progress-monitor .snapshot-preview:hover {
-    border-color: #58a6ff;
-    box-shadow: 0 0 0 1px rgba(88, 166, 255, 0.2);
-}
-#progress-monitor .snapshot-preview img {
-    width: 100%;
-    height: 100%;
-    object-fit: cover;
-    object-position: top center;
-    display: block;
-}
-#progress-monitor .snapshot-preview.placeholder {
-    font-size: 20px;
-}
-#progress-monitor .snapshot-info {
-    flex: 1;
-    min-width: 0;
-}
-#progress-monitor .snapshot-title-line {
-    display: flex;
-    align-items: center;
-    gap: 7px;
-    min-width: 0;
-}
-#progress-monitor .snapshot-favicon {
-    width: 16px;
-    height: 16px;
-    border-radius: 3px;
-    object-fit: contain;
-    background: #fff;
-    flex: 0 0 16px;
-}
-#progress-monitor .snapshot-title {
-    color: #f0f6fc;
-    font-size: 12px;
-    font-weight: 600;
-    line-height: 1.3;
-    white-space: nowrap;
-    overflow: hidden;
-    text-overflow: ellipsis;
-}
-#progress-monitor .snapshot-url {
-    font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
-    font-size: 11px;
-    color: #c9d1d9;
-    white-space: nowrap;
-    overflow: hidden;
-    text-overflow: ellipsis;
-}
-#progress-monitor .snapshot-meta {
-    font-size: 10px;
-    color: #8b949e;
-    margin-top: 2px;
-}
-#progress-monitor .snapshot-progress {
-    padding: 0 12px 8px;
-}
-
-/* Extractor List - Compact Badge Layout */
-#progress-monitor .extractor-list {
-    padding: 8px 12px;
-    background: rgba(0,0,0,0.2);
-    border-top: 1px solid #21262d;
-    display: flex;
-    flex-wrap: wrap;
-    gap: 4px;
-}
-#progress-monitor .extractor-badge {
-    position: relative;
-    display: inline-flex;
-    align-items: center;
-    gap: 4px;
-    padding: 3px 8px;
-    border-radius: 4px;
-    font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
-    font-size: 10px;
-    background: #21262d;
-    overflow: hidden;
-    white-space: nowrap;
-    color: inherit;
-    text-decoration: none;
-}
-#progress-monitor a.extractor-badge:hover {
-    background: #30363d;
-    color: #f0f6fc;
-}
-#progress-monitor .extractor-badge .progress-fill {
-    position: absolute;
-    top: 0;
-    left: 0;
-    bottom: 0;
-    z-index: 0;
-    transition: width 0.3s ease-out;
-}
-#progress-monitor .extractor-badge .badge-content {
-    position: relative;
-    z-index: 1;
-    display: flex;
-    align-items: center;
-    gap: 4px;
-}
-#progress-monitor .extractor-badge.queued {
-    color: #8b949e;
-}
-#progress-monitor .extractor-badge.queued .progress-fill {
-    background: rgba(110, 118, 129, 0.2);
-    width: 0%;
-}
-#progress-monitor .extractor-badge.started {
-    color: #d29922;
-}
-#progress-monitor .extractor-badge.started .progress-fill {
-    background: rgba(210, 153, 34, 0.3);
-    animation: progress-pulse 1.5s ease-in-out infinite;
-}
-@keyframes progress-pulse {
-    0%, 100% { opacity: 0.5; }
-    50% { opacity: 1; }
-}
-#progress-monitor .extractor-badge.succeeded {
-    color: #3fb950;
-}
-#progress-monitor .extractor-badge.succeeded .progress-fill {
-    background: rgba(63, 185, 80, 0.25);
-    width: 100%;
-}
-#progress-monitor .extractor-badge.failed {
-    color: #f85149;
-}
-#progress-monitor .extractor-badge.failed .progress-fill {
-    background: rgba(248, 81, 73, 0.25);
-    width: 100%;
-}
-#progress-monitor .extractor-badge.backoff {
-    color: #b8860b;
-}
-#progress-monitor .extractor-badge.backoff .progress-fill {
-    background: rgba(210, 153, 34, 0.2);
-    width: 30%;
-}
-#progress-monitor .extractor-badge.skipped {
-    color: #6e7681;
-}
-#progress-monitor .extractor-badge.skipped .progress-fill {
-    background: rgba(110, 118, 129, 0.15);
-    width: 100%;
-}
-#progress-monitor .extractor-badge .badge-icon {
-    font-size: 10px;
-}
-/* Status Badge */
-#progress-monitor .status-badge {
-    font-size: 10px;
-    padding: 2px 6px;
-    border-radius: 10px;
-    font-weight: 500;
-    text-transform: uppercase;
-    letter-spacing: 0.3px;
-}
-#progress-monitor .status-badge.queued {
-    background: #21262d;
-    color: #8b949e;
-}
-#progress-monitor .status-badge.started {
-    background: rgba(210, 153, 34, 0.2);
-    color: #d29922;
-}
-#progress-monitor .status-badge.paused {
-    background: rgba(88, 166, 255, 0.18);
-    color: #58a6ff;
-}
-#progress-monitor .status-badge.sealed,
-#progress-monitor .status-badge.succeeded {
-    background: rgba(63, 185, 80, 0.2);
-    color: #3fb950;
-}
-#progress-monitor .status-badge.failed {
-    background: rgba(248, 81, 73, 0.2);
-    color: #f85149;
-}
-#progress-monitor .status-badge.backoff {
-    background: rgba(210, 153, 34, 0.15);
-    color: #b8860b;
-}
-#progress-monitor .status-badge.unknown {
-    background: #21262d;
-    color: #6e7681;
-}
-#progress-monitor .duration-badge {
-    flex-shrink: 0;
-    min-width: 28px;
-    padding: 2px 6px;
-    border-radius: 10px;
-    background: rgba(88, 166, 255, 0.12);
-    color: #a5d6ff;
-    border: 1px solid rgba(88, 166, 255, 0.2);
-    font-size: 10px;
-    font-weight: 600;
-    line-height: 1.2;
-    text-align: center;
-    font-variant-numeric: tabular-nums;
-    white-space: nowrap;
-}
-
-#progress-monitor .pid-label {
-    display: inline-flex;
-    align-items: center;
-    gap: 4px;
-    padding: 2px 6px;
-    border-radius: 999px;
-    font-size: 10px;
-    font-weight: 600;
-    color: #8b949e;
-    background: rgba(148, 163, 184, 0.12);
-    border: 1px solid rgba(148, 163, 184, 0.2);
-    font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
-    letter-spacing: 0.2px;
-    white-space: nowrap;
-}
-#progress-monitor .pid-label.compact {
-    padding: 1px 5px;
-    font-size: 9px;
-}
-
-@media (max-width: 900px) {
-    #progress-monitor .header-bar {
-        align-items: flex-start;
-        gap: 8px;
-    }
-    #progress-monitor .header-left {
-        flex: 1;
-        min-width: 0;
-        flex-wrap: wrap;
-        gap: 8px 14px;
-    }
-    #progress-monitor .stats {
-        flex-wrap: wrap;
-        gap: 8px 12px;
-    }
-    #progress-monitor .tree-container {
-        max-height: 310px;
-    }
-    #progress-monitor .progress-content {
-        flex-direction: column;
-        padding: 10px 8px;
-    }
-    #progress-monitor .screencast-panel {
-        position: static;
-        order: -1;
-        width: min(100%, 336px);
-        flex-basis: auto;
-        align-self: center;
-    }
-    #progress-monitor .crawl-header {
-        align-items: flex-start;
-        gap: 8px;
-        padding: 8px;
-    }
-    #progress-monitor .crawl-header-link {
-        display: grid;
-        grid-template-columns: minmax(0, 1fr) auto auto auto;
-        align-items: start;
-        gap: 8px;
-        width: 100%;
-    }
-    #progress-monitor .crawl-info {
-        grid-column: 1 / -1;
-        width: 100%;
-    }
-    #progress-monitor .crawl-label {
-        max-width: 100%;
-    }
-    #progress-monitor .crawl-badges,
-    #progress-monitor .crawl-stats {
-        justify-content: flex-start;
-        min-width: 0;
-    }
-    #progress-monitor .crawl-badge {
-        max-width: min(100%, 260px);
-    }
-    #progress-monitor .crawl-stats {
-        grid-column: 1 / 2;
-    }
-    #progress-monitor .crawl-body {
-        padding: 0 8px 10px;
-    }
-    #progress-monitor .crawl-progress {
-        padding: 8px;
-    }
-    #progress-monitor .snapshot-header {
-        align-items: flex-start;
-        padding: 8px;
-    }
-    #progress-monitor .snapshot-header-link {
-        min-width: 0;
-    }
-}
-
-@media (max-width: 520px) {
-    #progress-monitor {
-        font-size: 11px;
-    }
-    #progress-monitor .header-bar {
-        display: grid;
-        grid-template-columns: minmax(0, 1fr) auto;
-        align-items: start;
-        padding: 7px 8px;
-    }
-    #progress-monitor .header-left {
-        gap: 6px 10px;
-    }
-    #progress-monitor .header-right {
-        justify-self: end;
-    }
-    #progress-monitor .toggle-btn {
-        white-space: nowrap;
-    }
-    #progress-monitor .orchestrator-status {
-        max-width: 100%;
-    }
-    #progress-monitor .stats {
-        flex: 1 0 100%;
-        gap: 6px 10px;
-    }
-    #progress-monitor .stat-label {
-        font-size: 9px;
-    }
-    #progress-monitor .tree-container {
-        max-height: 300px;
-    }
-    #progress-monitor .progress-content {
-        padding: 8px;
-    }
-    #progress-monitor .screencast-panel {
-        width: 100%;
-        align-self: stretch;
-    }
-    #progress-monitor .screencast-frame {
-        height: 300px;
-    }
-    #progress-monitor .crawl-header {
-        display: grid;
-        grid-template-columns: minmax(0, 1fr) auto;
-        padding: 8px;
-    }
-    #progress-monitor .crawl-header-link {
-        grid-template-columns: minmax(0, 1fr);
-    }
-    #progress-monitor .crawl-stats {
-        grid-column: auto;
-    }
-    #progress-monitor .crawl-title-link {
-        white-space: normal;
-        line-height: 1.35;
-    }
-    #progress-monitor .crawl-badges {
-        gap: 4px;
-    }
-    #progress-monitor .crawl-badge {
-        max-width: 100%;
-        min-width: 0;
-        font-size: 10px;
-        white-space: nowrap;
-    }
-    #progress-monitor .crawl-stats {
-        gap: 4px;
-    }
-    #progress-monitor .status-badge,
-    #progress-monitor .duration-badge,
-    #progress-monitor .pid-label {
-        justify-self: start;
-    }
-    #progress-monitor .snapshot-header-link {
-        display: grid;
-        grid-template-columns: 56px minmax(0, 1fr);
-        gap: 8px;
-    }
-    #progress-monitor .snapshot-preview {
-        width: 56px;
-        height: 42px;
-        flex-basis: 56px;
-    }
-    #progress-monitor .snapshot-title-line {
-        align-items: flex-start;
-    }
-    #progress-monitor .snapshot-title,
-    #progress-monitor .snapshot-url {
-        white-space: normal;
-        overflow-wrap: anywhere;
-    }
-    #progress-monitor .extractor-list {
-        padding: 7px 8px;
-        gap: 4px;
-    }
-    #progress-monitor .extractor-badge {
-        max-width: 100%;
-        white-space: nowrap;
-    }
-}
-
-#progress-monitor .crawl-notice {
-    padding: 8px 14px;
-    font-size: 11px;
-    color: var(--notice-color);
-    border-top: 1px solid var(--notice-border, var(--notice-color));
-    background: var(--notice-background);
-}
-#progress-monitor .crawl-notice.error {
-    --notice-color: #f85149;
-    --notice-background: rgba(248, 81, 73, 0.1);
-}
-#progress-monitor .crawl-notice.warning {
-    --notice-color: #d29922;
-    --notice-background: rgba(210, 153, 34, 0.1);
-}
-#progress-monitor .crawl-notice.info {
-    --notice-color: #58a6ff;
-    --notice-background: rgba(88, 166, 255, 0.1);
-}
-#progress-monitor .crawl-notice.cancelled {
-    --notice-color: #8b949e;
-    --notice-border: #6e7681;
-    --notice-background: rgba(139, 148, 158, 0.1);
-}

archivebox/progressmonitor/static/progressmonitor/monitor.js

@@ -1,860 +0,0 @@
-(function() {
-    const monitor = document.getElementById('progress-monitor');
-    const collapseBtn = document.getElementById('progress-collapse');
-    const treeContainer = document.getElementById('tree-container');
-    const crawlTree = document.getElementById('crawl-tree');
-    const idleMessage = document.getElementById('idle-message');
-    const screencastPanel = document.getElementById('screencast-panel');
-
-    let pollInterval = null;
-    let pollDelayMs = 1000;
-    let idleTicks = 0;
-    let isCollapsed = monitor.dataset.autoExpand !== '1';
-    const snapshotMedia = new Map();
-
-    function stableSnapshotMedia(snapshot) {
-        // The progress endpoint can briefly omit media URLs while ArchiveResult
-        // rows settle. Once this page has seen a snapshot image URL, keep using
-        // it so the preview/favicon branch does not flicker back to placeholder.
-        const snapshotId = String(snapshot.id || '');
-        const media = snapshotMedia.get(snapshotId) || {};
-        if (snapshot.favicon_url) media.favicon_url ||= snapshot.favicon_url;
-        if (snapshot.preview_url) {
-            media.preview_url ||= snapshot.preview_url;
-            media.preview_link ||= snapshot.preview_link;
-            media.preview_fallbacks ||= snapshot.preview_fallbacks || [];
-        }
-        if (snapshotId) snapshotMedia.set(snapshotId, media);
-        return media;
-    }
-    function preserveStableMediaNodes(nextRoot) {
-        // Snapshot preview/favicon files are immutable once visible. Keep the
-        // loaded DOM nodes across poll renders so the browser never reloads
-        // them while progress text/badges continue updating around them.
-        const currentCards = new Map(
-            Array.from(crawlTree.querySelectorAll('.snapshot-item[data-snapshot-id]')).map(card => [card.dataset.snapshotId, card])
-        );
-        nextRoot.querySelectorAll('.snapshot-item[data-snapshot-id]').forEach(nextCard => {
-            const currentCard = currentCards.get(nextCard.dataset.snapshotId);
-            if (!currentCard) return;
-            currentCard.querySelectorAll('[data-stable-media]').forEach(currentNode => {
-                const nextNode = nextCard.querySelector(`[data-stable-media="${currentNode.dataset.stableMedia}"]`);
-                if (nextNode) nextNode.replaceWith(currentNode);
-            });
-        });
-    }
-    function replaceCrawlTree(html) {
-        const template = document.createElement('template');
-        template.innerHTML = html;
-        preserveStableMediaNodes(template.content);
-        crawlTree.replaceChildren(...template.content.childNodes);
-    }
-    window.nextPreviewFallback = function(img) {
-        const fallbacks = (img.dataset.fallbacks || '').split(',').filter(Boolean);
-        if (fallbacks.length > 0) {
-            img.src = fallbacks.shift();
-            img.dataset.fallbacks = fallbacks.join(',');
-        } else {
-            const preview = img.closest('.snapshot-preview');
-            if (preview) {
-                preview.removeAttribute('data-stable-media');
-                preview.classList.add('placeholder');
-                preview.innerHTML = '<span>▤</span>';
-            }
-        }
-    };
-    function formatUrl(url) {
-        if (!url) return '(no URL)';
-        try {
-            const u = new URL(url);
-            return u.hostname + u.pathname.substring(0, 30) + (u.pathname.length > 30 ? '...' : '');
-        } catch {
-            return String(url).substring(0, 50) + (String(url).length > 50 ? '...' : '');
-        }
-    }
-
-    function getPluginIcon(plugin) {
-        const icons = {
-            'screenshot': '▧',
-            'chrome_mhtml': '▧',
-            'favicon': '◆',
-            'dom': '&lt;/&gt;',
-            'pdf': 'PDF',
-            'title': 'T',
-            'headers': '{}',
-            'singlefile': '▣',
-            'readability': 'R',
-            'mercury': 'M',
-            'wget': '↓',
-            'media': '▶',
-        };
-        return icons[plugin] || '•';
-    }
-
-    function formatDuration(seconds) {
-        seconds = Math.max(0, Math.floor(seconds || 0));
-        const hours = Math.floor(seconds / 3600);
-        const minutes = Math.floor((seconds % 3600) / 60);
-        const secs = seconds % 60;
-        if (hours > 0) return `${hours}hr ${minutes}m ${secs}s`;
-        if (minutes > 0) return `${minutes}m ${secs}s`;
-        return `${secs}s`;
-    }
-
-    function durationText(startedAt) {
-        const startedMs = Date.parse(startedAt || '');
-        if (!Number.isFinite(startedMs)) return '';
-        return formatDuration((Date.now() - startedMs) / 1000);
-    }
-
-    function renderDurationBadge(startedAt) {
-        const text = durationText(startedAt);
-        if (!text) return '';
-        return `<span class="duration-badge" data-started-at="${ArchiveBoxUI.escapeHtml(startedAt)}" title="Duration">${ArchiveBoxUI.escapeHtml(text)}</span>`;
-    }
-
-    function updateDurationBadges() {
-        document.querySelectorAll('#progress-monitor .duration-badge[data-started-at]').forEach((badge) => {
-            const text = durationText(badge.dataset.startedAt);
-            if (text) badge.textContent = text;
-        });
-    }
-
-    function activeScreencastTarget(data) {
-        const hookText = (item) => `${item.plugin || ''} ${item.hook_name || ''} ${item.label || ''}`.toLowerCase();
-        let fallback = null;
-        for (const crawl of data.active_crawls || []) {
-            for (const snapshot of crawl.active_snapshots || []) {
-                if (!Array.isArray(snapshot) && snapshot.screencast_url) {
-                    return {type: 'frame', crawl, snapshot};
-                }
-            }
-        }
-        for (const crawl of data.active_crawls || []) {
-            if (fallback) continue;
-
-            const setupPlugins = crawl.setup_plugins || [];
-            const browserReady = setupPlugins.some((item) => (
-                item.status === 'succeeded' && hookText(item).includes('chrome wait')
-            ));
-            if (!browserReady && setupPlugins.some((item) => item.status === 'started' && hookText(item).includes('chrome launch'))) {
-                fallback = {
-                    type: 'placeholder',
-                    crawl,
-                    state: 'launching',
-                    icon: '▣',
-                    title: 'Launching browser',
-                    subtitle: 'Chrome is starting and publishing its CDP session.',
-                };
-                continue;
-            }
-            const openingSnapshot = (crawl.active_snapshots || []).find((snapshot) => {
-                if (Array.isArray(snapshot)) return false;
-                const plugins = snapshot.all_plugins || [];
-                const tabReady = plugins.some((item) => (
-                    item.status === 'succeeded' && hookText(item).includes('chrome wait')
-                ));
-                return !tabReady && plugins.some((item) => (
-                    item.status === 'started' && hookText(item).includes('chrome tab')
-                ));
-            });
-            if (openingSnapshot) {
-                fallback = {
-                    type: 'placeholder',
-                    crawl,
-                    snapshot: openingSnapshot,
-                    state: 'launching',
-                    icon: '▣',
-                    title: 'Opening browser tab',
-                    subtitle: 'The snapshot tab is being attached for capture.',
-                };
-                continue;
-            }
-            if ((crawl.status === 'queued' || crawl.status === 'started') && !setupPlugins.some((item) => hookText(item).includes('chrome launch'))) {
-                fallback = {
-                    type: 'placeholder',
-                    crawl,
-                    state: 'not-launched',
-                    icon: '▯',
-                    title: 'Browser not launched yet',
-                    subtitle: 'Waiting for the crawl setup hooks to start Chrome.',
-                };
-            }
-        }
-        return fallback;
-    }
-
-    function updateScreencastPanel(data) {
-        const hasWork = (data.active_crawls || []).length > 0 ||
-            (data.crawls_queued || 0) > 0 ||
-            (data.crawls_active || 0) > 0 ||
-            (data.snapshots_queued || 0) > 0 ||
-            (data.snapshots_active || 0) > 0;
-        let target = activeScreencastTarget(data);
-        if (!target) {
-            const currentFrame = screencastPanel.querySelector('.screencast-frame img');
-            if (
-                hasWork &&
-                screencastPanel.dataset.mode === 'frame' &&
-                currentFrame?.getAttribute('src') &&
-                currentFrame.complete &&
-                currentFrame.naturalWidth > 0
-            ) {
-                screencastPanel.classList.add('visible');
-                currentFrame.style.opacity = '1';
-                return;
-            }
-            if (!hasWork) {
-                screencastPanel.classList.remove('visible');
-                screencastPanel.dataset.mode = '';
-                screencastPanel.innerHTML = '';
-                return;
-            }
-            target = {
-                type: 'placeholder',
-                state: 'not-launched',
-                icon: '▯',
-                title: 'Waiting for browser',
-                subtitle: 'Live preview will appear whenever Chrome is active.',
-            };
-        }
-        if (target.type === 'placeholder') {
-            const currentFrame = screencastPanel.querySelector('.screencast-frame img');
-            if (
-                screencastPanel.dataset.mode === 'frame' &&
-                currentFrame?.getAttribute('src') &&
-                currentFrame.complete &&
-                currentFrame.naturalWidth > 0
-            ) {
-                screencastPanel.classList.add('visible');
-                currentFrame.style.opacity = '1';
-                return;
-            }
-            const crawlUrl = target.crawl?.id ? `/admin/crawls/crawl/${target.crawl.id}/change/` : '/admin/';
-            const placeholderKey = [target.state, target.title, target.subtitle, crawlUrl].join('|');
-            if (screencastPanel.dataset.mode === 'placeholder' && screencastPanel.dataset.key === placeholderKey) {
-                return;
-            }
-            screencastPanel.dataset.mode = 'placeholder';
-            screencastPanel.dataset.key = placeholderKey;
-            screencastPanel.classList.add('visible');
-            screencastPanel.innerHTML = `
-                <a class="screencast-frame" href="${ArchiveBoxUI.escapeHtml(crawlUrl)}" title="Open crawl admin">
-                    <span class="screencast-placeholder ${ArchiveBoxUI.escapeHtml(target.state)}">
-                        <span class="screencast-placeholder-icon">${ArchiveBoxUI.escapeHtml(target.icon)}</span>
-                        <span class="screencast-placeholder-title">${ArchiveBoxUI.escapeHtml(target.title)}</span>
-                        <span class="screencast-placeholder-subtitle">${ArchiveBoxUI.escapeHtml(target.subtitle)}</span>
-                    </span>
-                </a>
-                <a class="screencast-caption" href="${ArchiveBoxUI.escapeHtml(crawlUrl)}" title="Open crawl admin">
-                    <span class="screencast-dot ${ArchiveBoxUI.escapeHtml(target.state)}"></span>
-                    <span class="screencast-text">
-                        <span class="screencast-title">${ArchiveBoxUI.escapeHtml(target.title)}</span>
-                        <span class="screencast-url">Crawl setup</span>
-                    </span>
-                </a>
-            `;
-            return;
-        }
-        const snapshot = target.snapshot;
-        const adminUrl = snapshot.admin_url || `/admin/core/snapshot/${snapshot.id || 'unknown'}/change/`;
-        const linkUrl = snapshot.screencast_link || snapshot.view_url || adminUrl;
-        const titleText = snapshot.title || formatUrl(snapshot.full_url || snapshot.url);
-        const urlText = snapshot.full_url || snapshot.url || '';
-        const newUrl = snapshot.screencast_url;
-        const snapshotKey = `${target.crawl?.id || ''}|${snapshot.id || ''}`;
-        screencastPanel.dataset.desiredFrameKey = snapshotKey;
-
-        if (screencastPanel.dataset.mode !== 'frame' && !screencastPanel.querySelector('.screencast-frame img')?.getAttribute('src')) {
-            if (screencastPanel.dataset.pendingFrameSrc) return;
-            screencastPanel.dataset.pendingFrameSrc = newUrl;
-            screencastPanel.dataset.pendingFrameKey = snapshotKey;
-            const firstFrame = new Image();
-            firstFrame.decoding = 'async';
-            firstFrame.onload = () => {
-                (firstFrame.decode ? firstFrame.decode() : Promise.resolve()).catch(() => {}).finally(() => {
-                    if (screencastPanel.dataset.pendingFrameSrc !== newUrl) return;
-                    delete screencastPanel.dataset.pendingFrameSrc;
-                    delete screencastPanel.dataset.pendingFrameKey;
-                    if (screencastPanel.dataset.desiredFrameKey !== snapshotKey) return;
-                    screencastPanel.dataset.mode = 'frame';
-                    screencastPanel.dataset.key = '';
-                    screencastPanel.dataset.currentFrameSrc = newUrl;
-                    screencastPanel.innerHTML = `
-                        <a class="screencast-frame" href="${ArchiveBoxUI.escapeHtml(linkUrl)}" title="Open active snapshot">
-                            <img src="${ArchiveBoxUI.escapeHtml(newUrl)}" alt="" decoding="async" loading="eager">
-                        </a>
-                        <a class="screencast-caption" href="${ArchiveBoxUI.escapeHtml(adminUrl)}" title="Open snapshot admin">
-                            <span class="screencast-dot"></span>
-                            <span class="screencast-text">
-                                <span class="screencast-title">${ArchiveBoxUI.escapeHtml(titleText)}</span>
-                                <span class="screencast-url">${ArchiveBoxUI.escapeHtml(urlText)}</span>
-                            </span>
-                        </a>
-                    `;
-                    screencastPanel.classList.add('visible');
-                });
-            };
-            firstFrame.onerror = () => {
-                if (screencastPanel.dataset.pendingFrameSrc === newUrl) {
-                    delete screencastPanel.dataset.pendingFrameSrc;
-                    delete screencastPanel.dataset.pendingFrameKey;
-                }
-            };
-            firstFrame.src = newUrl;
-            return;
-        }
-
-        if (screencastPanel.dataset.mode !== 'frame') {
-            screencastPanel.dataset.mode = 'frame';
-            screencastPanel.dataset.key = '';
-            screencastPanel.innerHTML = `
-                <a class="screencast-frame" href="" title="Open active snapshot">
-                    <img src="" alt="" decoding="async" loading="eager">
-                </a>
-                <a class="screencast-caption" href="" title="Open snapshot admin">
-                    <span class="screencast-dot"></span>
-                    <span class="screencast-text">
-                        <span class="screencast-title"></span>
-                        <span class="screencast-url"></span>
-                    </span>
-                </a>
-            `;
-        }
-        const frameLink = screencastPanel.querySelector('.screencast-frame');
-        const captionLink = screencastPanel.querySelector('.screencast-caption');
-        const img = screencastPanel.querySelector('.screencast-frame img');
-        const titleEl = screencastPanel.querySelector('.screencast-title');
-        const urlEl = screencastPanel.querySelector('.screencast-url');
-        frameLink.href = linkUrl;
-        captionLink.href = adminUrl;
-        titleEl.textContent = titleText;
-        urlEl.textContent = urlText;
-        if (img.getAttribute('src')) {
-            img.style.opacity = '1';
-            screencastPanel.classList.add('visible');
-        }
-        if (screencastPanel.dataset.currentFrameSrc !== newUrl && screencastPanel.dataset.pendingFrameSrc !== newUrl && !screencastPanel.dataset.pendingFrameSrc) {
-            screencastPanel.dataset.pendingFrameSrc = newUrl;
-            screencastPanel.dataset.pendingFrameKey = snapshotKey;
-            const nextFrame = new Image();
-            nextFrame.decoding = 'async';
-            nextFrame.onload = () => {
-                (nextFrame.decode ? nextFrame.decode() : Promise.resolve()).catch(() => {}).finally(() => {
-                    if (screencastPanel.dataset.pendingFrameSrc !== newUrl) return;
-                    delete screencastPanel.dataset.pendingFrameSrc;
-                    delete screencastPanel.dataset.pendingFrameKey;
-                    if (screencastPanel.dataset.desiredFrameKey !== snapshotKey) return;
-                    const currentImg = screencastPanel.querySelector('.screencast-frame img');
-                    if (!currentImg) return;
-                    nextFrame.alt = '';
-                    nextFrame.loading = 'eager';
-                    nextFrame.style.opacity = '1';
-                    currentImg.replaceWith(nextFrame);
-                    screencastPanel.dataset.currentFrameSrc = newUrl;
-                    screencastPanel.classList.add('visible');
-                });
-            };
-            nextFrame.onerror = () => {
-                if (screencastPanel.dataset.pendingFrameSrc === newUrl) {
-                    delete screencastPanel.dataset.pendingFrameSrc;
-                    delete screencastPanel.dataset.pendingFrameKey;
-                    if (!screencastPanel.querySelector('.screencast-frame img')?.getAttribute('src')) {
-                        screencastPanel.classList.remove('visible');
-                    }
-                }
-            };
-            nextFrame.src = newUrl;
-        }
-    }
-
-    function renderExtractor(extractor) {
-        const statusIcons = {started: '▶', succeeded: '✓', failed: '!', backoff: 'wait', skipped: 'skip', noresults: '∅'};
-        const icon = statusIcons[extractor.status] || getPluginIcon(extractor.plugin);
-        const progress = typeof extractor.progress === 'number'
-            ? Math.max(0, Math.min(100, extractor.progress))
-            : null;
-        const progressStyle = progress !== null ? ` style="width: ${progress}%;"` : '';
-        const pidHtml = extractor.status === 'started' && extractor.pid ? `<span class="pid-label compact">pid ${extractor.pid}</span>` : '';
-        const href = extractor.output_url || extractor.admin_url || '';
-        const tag = href ? 'a' : 'span';
-        const hrefAttr = href ? ` href="${ArchiveBoxUI.escapeHtml(href)}"` : '';
-        const title = extractor.output_path
-            ? `${extractor.plugin || 'output'}: ${extractor.output_path}`
-            : `${extractor.plugin || 'hook'}${extractor.hook_name ? `: ${extractor.hook_name}` : ''}`;
-
-        return `
-            <${tag} class="extractor-badge ${extractor.status || 'queued'}"${hrefAttr} title="${ArchiveBoxUI.escapeHtml(title)}">
-                <span class="progress-fill"${progressStyle}></span>
-                <span class="badge-content">
-                    <span class="badge-icon">${icon}</span>
-                    <span>${ArchiveBoxUI.escapeHtml(extractor.label || extractor.plugin || 'unknown')}</span>
-                    ${pidHtml}
-                </span>
-            </${tag}>
-        `;
-    }
-
-    function renderExtractors(extractors = []) {
-        if (!extractors.length) return '';
-        const sorted = [...extractors].sort((a, b) => (a.plugin || '').localeCompare(b.plugin || ''));
-        return `<div class="extractor-list">${sorted.map(renderExtractor).join('')}</div>`;
-    }
-
-    function renderSnapshot(snapshot) {
-        if (Array.isArray(snapshot)) {
-            snapshot = {
-                id: snapshot[0],
-                url: snapshot[1],
-                title: snapshot[2],
-                status: snapshot[3] || 'queued',
-            };
-        }
-        const statusIcon = snapshot.status === 'started' ? '▤' : '▢';
-        const adminUrl = snapshot.admin_url || `/admin/core/snapshot/${snapshot.id || 'unknown'}/change/`;
-        const canCancel = snapshot.status === 'queued';
-        const cancelBtn = canCancel
-            ? `<button class="cancel-item-btn" data-cancel-type="snapshot" data-snapshot-id="${snapshot.id}" data-label="✕" title="Cancel snapshot">✕</button>`
-            : '';
-        const snapshotPidHtml = snapshot.worker_pid ? `<span class="pid-label compact">pid ${snapshot.worker_pid}</span>` : '';
-        const snapshotDurationHtml = renderDurationBadge(snapshot.started);
-        const titleText = snapshot.title || formatUrl(snapshot.full_url || snapshot.url);
-        const urlText = snapshot.full_url || snapshot.url || '';
-        const media = stableSnapshotMedia(snapshot);
-        const faviconHtml = media.favicon_url
-            ? `<img class="snapshot-favicon" data-stable-media="favicon" src="${ArchiveBoxUI.escapeHtml(media.favicon_url)}" alt="" decoding="async" loading="lazy" onerror="this.remove()">`
-            : '';
-        const previewHtml = media.preview_url
-            ? `<a class="snapshot-preview" data-stable-media="preview" href="${ArchiveBoxUI.escapeHtml(media.preview_link || snapshot.view_url || adminUrl)}" title="Open snapshot output"><img src="${ArchiveBoxUI.escapeHtml(media.preview_url)}" alt="" decoding="async" loading="lazy" data-fallbacks="${ArchiveBoxUI.escapeHtml((media.preview_fallbacks || []).join(','))}" onerror="nextPreviewFallback(this)"></a>`
-            : `<a class="snapshot-preview placeholder" href="${ArchiveBoxUI.escapeHtml(snapshot.view_url || adminUrl)}" title="Open snapshot"><span>${statusIcon}</span></a>`;
-
-        const extractorHtml = renderExtractors(snapshot.all_plugins);
-
-        const hasProcessEntries = (snapshot.all_plugins || []).some(extractor => extractor.source === 'process');
-        const hasArchiveResults = (snapshot.all_plugins || []).some(extractor => extractor.source === 'archiveresult');
-        const processOnly = hasProcessEntries && !hasArchiveResults;
-        const runningProcessCount = (snapshot.all_plugins || []).filter(extractor => extractor.source === 'process' && extractor.status === 'started').length;
-        const failedProcessCount = (snapshot.all_plugins || []).filter(extractor => extractor.source === 'process' && extractor.status === 'failed').length;
-        const snapshotMeta = (snapshot.total_plugins || 0) > 0
-            ? processOnly
-                ? runningProcessCount > 0
-                    ? `Running ${runningProcessCount}/${snapshot.total_plugins || 0} setup hooks`
-                    : failedProcessCount > 0
-                        ? `${failedProcessCount} setup hook${failedProcessCount === 1 ? '' : 's'} failed`
-                        : `${snapshot.completed_plugins || 0}/${snapshot.total_plugins || 0} setup hooks`
-                : hasProcessEntries
-                    ? `${snapshot.completed_plugins || 0}/${snapshot.total_plugins || 0} tasks${(snapshot.failed_plugins || 0) > 0 ? ` <span style="color:#f85149">(${snapshot.failed_plugins} failed)</span>` : ''}${runningProcessCount > 0 ? ` <span style="color:#d29922">(${runningProcessCount} hooks running)</span>` : ''}`
-                    : `${snapshot.completed_plugins || 0}/${snapshot.total_plugins || 0} extractors${(snapshot.failed_plugins || 0) > 0 ? ` <span style="color:#f85149">(${snapshot.failed_plugins} failed)</span>` : ''}`
-            : snapshot.worker_state === 'crashed'
-                ? '<span style="color:#f85149">Worker stopped before extractors started</span>'
-                : snapshot.worker_state === 'stalled'
-                    ? '<span style="color:#d29922">Waiting for runner to resume</span>'
-                    : snapshot.worker_state === 'cancelled'
-                        ? '<span style="color:#8b949e">Cancelled before completion</span>'
-                        : 'Waiting for extractors...';
-
-        return `
-            <div class="snapshot-item" data-snapshot-id="${ArchiveBoxUI.escapeHtml(snapshot.id || '')}">
-                <div class="snapshot-header">
-                    ${previewHtml}
-                    <a class="snapshot-header-link" href="${adminUrl}">
-                        <div class="snapshot-info">
-                            <div class="snapshot-title-line">
-                                ${faviconHtml}
-                                <span class="snapshot-title">${ArchiveBoxUI.escapeHtml(titleText)}</span>
-                            </div>
-                            <div class="snapshot-url">${ArchiveBoxUI.escapeHtml(urlText)}</div>
-                            <div class="snapshot-meta">
-                                ${snapshotMeta}
-                            </div>
-                        </div>
-                        ${snapshotPidHtml}
-                        ${snapshotDurationHtml}
-                        <span class="status-badge ${snapshot.status || 'unknown'}">${snapshot.status || 'unknown'}</span>
-                    </a>
-                    ${cancelBtn}
-                </div>
-                <div class="snapshot-progress">
-                    <div class="progress-bar-container">
-                        <div class="progress-bar snapshot ${((processOnly && runningProcessCount > 0) || (snapshot.status === 'started' && (snapshot.progress || 0) === 0)) ? 'indeterminate' : ''}"
-                             style="width: ${snapshot.progress || 0}%"></div>
-                    </div>
-                </div>
-                ${extractorHtml}
-            </div>
-        `;
-    }
-
-    function renderCrawl(crawl) {
-        const adminUrl = `/admin/crawls/crawl/${crawl.id || 'unknown'}/change/`;
-        const adminFieldUrl = (fieldName) => `${adminUrl}#id_${fieldName}`;
-        const crawlBadge = (className, label, value, href, title) => {
-            const tag = href ? 'a' : 'span';
-            const hrefAttr = href ? ` href="${ArchiveBoxUI.escapeHtml(href)}"` : '';
-            const titleAttr = title ? ` title="${ArchiveBoxUI.escapeHtml(title)}"` : '';
-            return `<${tag} class="crawl-badge ${className}"${hrefAttr}${titleAttr}><strong>${ArchiveBoxUI.escapeHtml(label)}</strong>${ArchiveBoxUI.escapeHtml(value)}</${tag}>`;
-        };
-        const crawlId = (crawl.id || 'unknown').toString();
-        const crawlShortId = crawlId === 'unknown' ? 'unknown' : crawlId.slice(-8);
-        const startedDate = crawl.started ? crawl.started.slice(0, 10) : 'unknown date';
-        const canCancel = crawl.status === 'queued' || crawl.status === 'started' || crawl.status === 'paused';
-        const canPause = (crawl.status === 'queued' || crawl.status === 'started') && !crawl.is_paused;
-        const canResume = crawl.status === 'paused' || crawl.is_paused;
-        const pauseBtn = canPause
-            ? `<button class="crawl-action-btn pause-item-btn" data-crawl-action="pause" data-crawl-id="${crawl.id}" data-label="Ⅱ" title="Pause crawl">Ⅱ</button>`
-            : canResume
-                ? `<button class="crawl-action-btn pause-item-btn" data-crawl-action="resume" data-crawl-id="${crawl.id}" data-label="▶" title="Resume crawl">▶</button>`
-                : '';
-        const cancelBtn = canCancel
-            ? `<button class="cancel-item-btn" data-cancel-type="crawl" data-crawl-id="${crawl.id}" data-label="✕" title="Cancel crawl">✕</button>`
-            : '';
-        const crawlPidHtml = crawl.worker_pid ? `<span class="pid-label compact">pid ${crawl.worker_pid}</span>` : '';
-        const crawlDurationHtml = renderDurationBadge(crawl.started);
-
-        const snapshotsHtml = (crawl.active_snapshots || []).map(renderSnapshot).join('');
-        const queuedSnapshotsHidden = Number(crawl.queued_snapshots_hidden || 0);
-        const queuedSnapshotsNote = queuedSnapshotsHidden > 0
-            ? `<div class="progress-overflow-note">${queuedSnapshotsHidden} more queued snapshot${queuedSnapshotsHidden === 1 ? '' : 's'} not shown</div>`
-            : '';
-        let setupHtml = '';
-        if (crawl.setup_plugins && crawl.setup_plugins.length > 0) {
-            setupHtml = `
-                <div class="snapshot-item">
-                    <div class="snapshot-header">
-                        <div class="snapshot-header-link">
-                            <span class="snapshot-icon">&#9881;</span>
-                            <div class="snapshot-info">
-                                <div class="snapshot-url">Crawl Setup</div>
-                            </div>
-                        </div>
-                    </div>
-                    ${renderExtractors(crawl.setup_plugins)}
-                </div>
-            `;
-        }
-
-        const urlPreview = crawl.urls_preview ? ` (${ArchiveBoxUI.escapeHtml(crawl.urls_preview)})` : '';
-        const warnings = [
-            [crawl.status === 'queued' && !crawl.can_start, 'error',
-                `Crawl cannot start: ${crawl.urls_preview ? 'unknown error' : 'no URLs'}`],
-            [crawl.is_paused, 'warning',
-                'Crawl is paused. Resume it to continue processing queued snapshots.'],
-            [crawl.status === 'queued' && crawl.retry_at_future, 'info',
-                `Trying in ${crawl.seconds_until_retry || 0}s...${urlPreview}`],
-            [crawl.status === 'queued' && crawl.total_snapshots === 0, 'warning',
-                `Waiting for the runner to pick up...${urlPreview}`],
-            [crawl.status === 'started' && crawl.worker_state === 'crashed', 'error',
-                `Runner stopped with ${crawl.started_snapshots || 0} active and ${crawl.pending_snapshots || 0} pending snapshots. It will resume when the runner starts again.`],
-            [crawl.worker_state === 'cancelled', 'cancelled',
-                `Crawl was cancelled. ${crawl.cancelled_snapshots || 0} snapshot${(crawl.cancelled_snapshots || 0) === 1 ? '' : 's'} stopped before completion.`],
-        ];
-        const warning = warnings.find(([matches]) => matches);
-        const warningHtml = warning ? `<div class="crawl-notice ${warning[1]}">${warning[2]}</div>` : '';
-
-        // Show crawl-scale limits and sealed output sizes from DB metadata.
-        const currentUrlCount = Math.max(crawl.total_snapshots || 0, crawl.urls_count || 0);
-        const maxUrlsText = (crawl.max_urls || 0) > 0 ? crawl.max_urls : 'unlimited';
-        const urlLimitText = `${currentUrlCount} / ${maxUrlsText}`;
-        const crawlSizeLimitText = `${crawl.crawl_output_size_display || '0 B'} / ${crawl.max_crawl_size_display || 'unlimited'}`;
-        const crawlTimeoutText = (crawl.crawl_timeout || 0) > 0 ? `${crawl.crawl_timeout}s` : 'unlimited';
-        const snapshotSizeLimitText = `${crawl.avg_snapshot_size_display || '0 B'} / ${crawl.max_snapshot_size_display || 'unlimited'}`;
-        const crawlBadges = [
-            crawlBadge('persona', 'Persona Config', crawl.persona || 'Default', crawl.persona_admin_url, 'Edit persona config'),
-            crawlBadge('limit', 'depth', crawl.max_depth || 0, adminFieldUrl('max_depth'), 'Edit crawl depth'),
-            crawlBadge('limit', 'urls', urlLimitText, adminFieldUrl('max_urls'), 'Edit max URLs'),
-            crawlBadge('size', 'crawl size', crawlSizeLimitText, adminFieldUrl('crawl_max_size'), 'Edit max crawl size'),
-            crawlBadge('limit', 'time', crawlTimeoutText, adminFieldUrl('crawl_timeout'), 'Edit max crawl time'),
-            crawlBadge('size', 'avg snap', snapshotSizeLimitText, adminFieldUrl('snapshot_max_size'), 'Edit max snapshot size'),
-            ...(crawl.tags || []).map(tag => `<a class="crawl-badge tag" href="${ArchiveBoxUI.escapeHtml(adminFieldUrl('tags_editor'))}" title="Edit crawl tags">#${ArchiveBoxUI.escapeHtml(tag)}</a>`),
-        ].join('');
-        const statsHtml = [
-            `<span class="crawl-badge count"><strong>done</strong>${crawl.completed_snapshots || 0}</span>`,
-            `<span class="crawl-badge size"><strong>active</strong>${crawl.started_snapshots || 0}</span>`,
-            `<span class="crawl-badge"><strong>pending</strong>${crawl.pending_snapshots || 0}</span>`,
-            (crawl.cancelled_snapshots || 0) > 0 ? `<span class="crawl-badge"><strong>cancelled</strong>${crawl.cancelled_snapshots}</span>` : '',
-        ].join('');
-
-        return `
-            <div class="crawl-item" data-crawl-id="${crawl.id || 'unknown'}">
-                <div class="crawl-header">
-                    <div class="crawl-header-link">
-                        <div class="crawl-info">
-                            <a class="crawl-label crawl-title-link" href="${ArchiveBoxUI.escapeHtml(adminUrl)}">Crawl #${ArchiveBoxUI.escapeHtml(crawlShortId)} ${ArchiveBoxUI.escapeHtml(startedDate)} started by ${ArchiveBoxUI.escapeHtml(crawl.created_by || 'unknown')}</a>
-                            <div class="crawl-badges">${crawlBadges}</div>
-                        </div>
-                        <div class="crawl-stats">
-                            ${statsHtml}
-                        </div>
-                        ${crawlPidHtml}
-                        ${crawlDurationHtml}
-                        <span class="status-badge ${crawl.status || 'unknown'}">${crawl.is_paused ? 'paused' : (crawl.status || 'unknown')}</span>
-                    </div>
-                    ${pauseBtn}
-                    ${cancelBtn}
-                </div>
-                <div class="crawl-progress">
-                    <div class="progress-bar-container">
-                        <div class="progress-bar crawl ${crawl.status === 'started' && (crawl.progress || 0) === 0 ? 'indeterminate' : ''}"
-                             style="width: ${crawl.progress || 0}%"></div>
-                    </div>
-                </div>
-                ${warningHtml}
-                <div class="crawl-body">
-                    <div class="snapshot-list">
-                        ${setupHtml}
-                        ${snapshotsHtml}
-                        ${queuedSnapshotsNote}
-                    </div>
-                </div>
-            </div>
-        `;
-    }
-
-    function setOrchestratorState(state, label) {
-        const dot = document.getElementById('orchestrator-dot');
-        dot.classList.remove('stopped', 'idle', 'running', 'error');
-        dot.classList.add(state);
-        document.getElementById('orchestrator-text').textContent = label;
-        return dot;
-    }
-
-    function updateProgress(data) {
-        idleMessage.style.color = '';
-        function setCount(id, value) {
-            const el = document.getElementById(id);
-            if (el) el.textContent = Number(value || 0).toLocaleString();
-        }
-        function setActiveCount(id, value) {
-            setCount(id, value);
-            const segment = document.getElementById(`${id}-segment`);
-            if (segment) segment.style.display = Number(value || 0) > 0 ? '' : 'none';
-        }
-
-        // Calculate if there's activity
-        const hasActivity = data.active_crawls.length > 0 ||
-                           data.crawls_queued > 0 || data.crawls_active > 0 ||
-                           data.snapshots_queued > 0 || data.snapshots_active > 0 ||
-                           data.archiveresults_queued > 0 || data.archiveresults_active > 0;
-        if (hasActivity) {
-            idleTicks = 0;
-            if (pollDelayMs !== 1000) {
-                setPollingDelay(1000);
-            }
-        } else {
-            idleTicks += 1;
-            if (idleTicks > 5 && pollDelayMs !== 10000) {
-                setPollingDelay(10000);
-            }
-        }
-
-        // Update orchestrator status - show "Running" only when there are active workers.
-        const pidEl = document.getElementById('orchestrator-pid');
-        const hasWorkers = data.total_workers > 0;
-        let dot = null;
-
-        if (hasWorkers) {
-            dot = setOrchestratorState('running', 'Running');
-        } else if (!data.orchestrator_running) {
-            dot = setOrchestratorState('stopped', 'Runner stopped');
-        } else if (hasActivity) {
-            dot = setOrchestratorState('idle', 'Idle');
-        } else {
-            dot = setOrchestratorState('idle', 'Idle');
-        }
-
-        if (data.orchestrator_pid) {
-            pidEl.textContent = `pid ${data.orchestrator_pid}`;
-            pidEl.style.display = 'inline-flex';
-        } else {
-            pidEl.textContent = '';
-            pidEl.style.display = 'none';
-        }
-
-        // Pulse the dot to show we got fresh data
-        dot.classList.add('flash');
-        setTimeout(() => dot.classList.remove('flash'), 300);
-
-        [
-            ['crawls-queued', data.crawls_queued],
-            ['snapshots-queued', data.snapshots_queued],
-            ['downloads-queued', data.downloads_queued],
-            ['indexing-queued', data.indexing_queued],
-        ].forEach(([id, value]) => setCount(id, value));
-        [
-            ['crawls-active', data.crawls_active],
-            ['snapshots-active', data.snapshots_active],
-            ['downloads-active', data.downloads_active],
-            ['indexing-active', data.indexing_active],
-        ].forEach(([id, value]) => setActiveCount(id, value));
-        updateScreencastPanel(data);
-
-        // Render crawl tree
-        if (data.active_crawls.length > 0) {
-            idleMessage.style.display = 'none';
-            const queuedCrawlsHidden = Number(data.queued_crawls_hidden || 0);
-            const queuedCrawlsNote = queuedCrawlsHidden > 0
-                ? `<div class="progress-overflow-note">${queuedCrawlsHidden} more queued crawl${queuedCrawlsHidden === 1 ? '' : 's'} not shown</div>`
-                : '';
-            replaceCrawlTree(data.active_crawls.map(c => renderCrawl(c)).join('') + queuedCrawlsNote);
-        } else if (hasActivity) {
-            idleMessage.style.display = 'none';
-            replaceCrawlTree(`
-                <div class="idle-message">
-                    ${data.snapshots_active || 0} snapshots processing, ${data.archiveresults_active || 0} extractors running
-                </div>
-            `);
-        } else {
-            idleMessage.style.display = '';
-            // Build the URL for recent crawls (last 24 hours)
-            var yesterday = new Date(Date.now() - 24*60*60*1000).toISOString().split('T')[0];
-            var recentUrl = '/admin/crawls/crawl/?created_at__gte=' + yesterday + '&o=-1';
-            idleMessage.innerHTML = `No active crawls (${data.crawls_queued || 0} pending, ${data.crawls_active || 0} started, <a href="${recentUrl}" style="color: #58a6ff;">${data.crawls_recent || 0} recent</a>)`;
-            replaceCrawlTree('');
-        }
-
-        updateDurationBadges();
-    }
-
-    const progressEndpoint = monitor.dataset.progressEndpoint || '/progress.json';
-
-    function fetchProgress() {
-        fetch(progressEndpoint, { credentials: 'same-origin' })
-            .then(response => {
-                if (!response.ok) throw new Error(`HTTP ${response.status}`);
-                return response.json();
-            })
-            .then(data => {
-                if (data.error) {
-                    console.error('Progress API error:', data.error, data.traceback);
-                    setOrchestratorState('error', 'Backend not responding');
-                    idleMessage.textContent = 'API Error: ' + data.error;
-                    idleMessage.style.color = '#f85149';
-                    return;
-                }
-                monitor.classList.toggle('is-guest', data.is_admin === false);
-                updateProgress(data);
-            })
-            .catch(error => {
-                console.error('Progress fetch error:', error);
-                setOrchestratorState('error', 'Backend not responding');
-                idleMessage.textContent = 'Fetch Error: ' + error.message;
-                idleMessage.style.color = '#f85149';
-            });
-    }
-
-    function startPolling() {
-        if (pollInterval) return;
-        fetchProgress();
-        pollInterval = setInterval(fetchProgress, pollDelayMs);
-    }
-
-    function stopPolling() {
-        if (pollInterval) {
-            clearInterval(pollInterval);
-            pollInterval = null;
-        }
-    }
-
-    function setPollingDelay(ms) {
-        pollDelayMs = ms;
-        if (pollInterval) {
-            clearInterval(pollInterval);
-            pollInterval = setInterval(fetchProgress, pollDelayMs);
-        }
-    }
-
-    function setCollapsedState(collapsed) {
-        isCollapsed = collapsed;
-        if (isCollapsed) {
-            monitor.classList.add('collapsed');
-            collapseBtn.textContent = 'Details';
-            collapseBtn.setAttribute('aria-expanded', 'false');
-        } else {
-            monitor.classList.remove('collapsed');
-            collapseBtn.textContent = 'Hide';
-            collapseBtn.setAttribute('aria-expanded', 'true');
-        }
-    }
-
-    function setActionButtonState(btn, busy) {
-        if (!btn) return;
-        const label = btn.dataset.label || '✕';
-        btn.disabled = !!busy;
-        btn.classList.toggle('is-busy', !!busy);
-        btn.textContent = busy ? '…' : label;
-    }
-
-    function patchProgressItem(url, action, btn, errorLabel) {
-        if (!url || !action) return;
-        setActionButtonState(btn, true);
-
-        fetch(ArchiveBoxUI.apiUrl(url), {
-            method: 'PATCH',
-            headers: ArchiveBoxUI.apiHeaders(),
-            credentials: 'same-origin',
-            body: JSON.stringify({ action: action }),
-        })
-        .then(response => response.json().then(data => ({response, data})))
-        .then(({response, data}) => {
-            if (!response.ok) throw new Error(data.detail || data.error || `HTTP ${response.status}`);
-            if (data.error) console.error(`${errorLabel} error:`, data.error);
-            fetchProgress();
-        })
-        .catch(error => {
-            console.error(`${errorLabel} failed:`, error);
-            setActionButtonState(btn, false);
-        });
-    }
-
-    // Collapse toggle
-    collapseBtn.addEventListener('click', function(event) {
-        event.stopPropagation();
-        setCollapsedState(!isCollapsed);
-    });
-    monitor.querySelector('.header-bar').addEventListener('click', function(event) {
-        if (event.target.closest('button, a, input, select, textarea, [role="button"]')) return;
-        if (isCollapsed) setCollapsedState(false);
-    });
-
-    crawlTree.addEventListener('click', function(event) {
-        const actionBtn = event.target.closest('.crawl-action-btn');
-        if (actionBtn) {
-            event.preventDefault();
-            event.stopPropagation();
-            patchProgressItem(
-                actionBtn.dataset.crawlId ? `/api/v1/crawls/crawl/${actionBtn.dataset.crawlId}` : '',
-                actionBtn.dataset.crawlAction,
-                actionBtn,
-                'Crawl action',
-            );
-            return;
-        }
-        const btn = event.target.closest('.cancel-item-btn');
-        if (!btn) return;
-        event.preventDefault();
-        event.stopPropagation();
-
-        const cancelType = btn.dataset.cancelType;
-        if (cancelType === 'crawl') {
-            patchProgressItem(btn.dataset.crawlId ? `/api/v1/crawls/crawl/${btn.dataset.crawlId}` : '', 'cancel', btn, 'Cancel crawl');
-        } else if (cancelType === 'snapshot') {
-            patchProgressItem(btn.dataset.snapshotId ? `/api/v1/core/snapshot/${btn.dataset.snapshotId}` : '', 'cancel', btn, 'Cancel snapshot');
-        }
-    });
-
-    // Apply initial state
-    setCollapsedState(isCollapsed);
-
-    // Start polling when page loads
-    startPolling();
-    setInterval(updateDurationBadges, 1000);
-
-    // Pause polling when tab is hidden
-    document.addEventListener('visibilitychange', function() {
-        if (document.hidden) {
-            stopPolling();
-        } else {
-            startPolling();
-        }
-    });
-})();

archivebox/progressmonitor/views.py

@@ -1,13 +1,23 @@
 __package__ = "archivebox.progressmonitor"
 
+from functools import lru_cache
+from pathlib import Path
 from typing import Literal
 
+from abx_dl.events import PROCESS_EXIT_SKIPPED
 from django.conf import settings
 from django.db import DatabaseError
+from django.db.models import CharField, Count, Q, Sum
+from django.db.models.functions import Cast
 from django.http import HttpResponse, JsonResponse
 from django.utils import timezone
 
+from archivebox.config import CONSTANTS
+from archivebox.config.common import get_config
 from archivebox.core.permissions import can_view_snapshot, is_admin_user
+from archivebox.core.routes_util import build_snapshot_url, build_web_url, get_api_base_url
+from archivebox.misc.logging_util import printable_filesize
+from archivebox.plugins.discovery import discover_plugin_configs
 
 
 def progress_endpoint(scope: Literal["crawl", "snapshot"] | None = None, object_id: object | None = None) -> str:
@@ -17,10 +27,28 @@ def progress_endpoint(scope: Literal["crawl", "snapshot"] | None = None, object_
     return f"/progress.json?{scope}_id={str(object_id).replace('-', '')}"
 
 
+@lru_cache(maxsize=1)
+def _live_progress_plugin_names() -> tuple[frozenset[str], frozenset[str]]:
+    plugin_configs = discover_plugin_configs()
+    indexing_plugin_names = frozenset(
+        plugin_name
+        for plugin_name, plugin_config in plugin_configs.items()
+        if {"search", "flush"}.issubset(plugin_config.get("commands", {}))
+    )
+    download_plugin_names = frozenset(
+        plugin_name
+        for plugin_name, plugin_config in plugin_configs.items()
+        if plugin_config.get("output_mimetypes") and plugin_name not in indexing_plugin_names
+    )
+    return download_plugin_names, indexing_plugin_names
+
+
 def live_progress_view(request):
     """Simple JSON endpoint for live progress status - used by admin progress monitor."""
     try:
-        from archivebox.core.models import Snapshot
+        from archivebox.core.models import ArchiveResult, Snapshot
+        from archivebox.crawls.models import Crawl
+        from archivebox.machine.models import Machine, Process
 
         snapshot_id_filter = (request.GET.get("snapshot_id") or "").strip().replace("-", "")
         crawl_id_filter = (request.GET.get("crawl_id") or "").strip().replace("-", "")
@@ -37,18 +65,887 @@ def live_progress_view(request):
             scoped_snapshot = Snapshot.objects.filter(id=snapshot_id_filter).select_related("crawl").first()
             if scoped_snapshot is None or not can_view_snapshot(request, scoped_snapshot):
                 return JsonResponse({"error": "Permission denied"}, status=403)
-        elif not is_admin:
-            # Crawl and global scopes require staff: a crawl can mix snapshot ACLs.
-            return JsonResponse({"error": "Permission denied"}, status=403)
+        elif crawl_id_filter:
+            # Crawl-only scope still requires staff: there's no per-crawl ACL helper,
+            # and a crawl can mix snapshot permissions levels.
+            if not is_admin:
+                return JsonResponse({"error": "Permission denied"}, status=403)
+        else:
+            if not is_admin:
+                return JsonResponse({"error": "Permission denied"}, status=403)
 
-        from .report import ProgressReport
+        request_config = request.archivebox_config
+        now = timezone.now()
+        crawl_scope = Crawl.objects.all()
+        snapshot_scope = Snapshot.objects.all()
+        archiveresult_scope = ArchiveResult.objects.all()
+        if is_admin and not request.user.is_superuser:
+            crawl_scope = crawl_scope.filter(created_by=request.user)
+            snapshot_scope = snapshot_scope.filter(crawl__created_by=request.user)
+            archiveresult_scope = archiveresult_scope.filter(snapshot__crawl__created_by=request.user)
+        if scoped_snapshot is not None:
+            snapshot_scope = Snapshot.objects.filter(id=scoped_snapshot.id)
+            crawl_scope = Crawl.objects.filter(id=scoped_snapshot.crawl_id)
+            archiveresult_scope = ArchiveResult.objects.filter(snapshot_id=scoped_snapshot.id)
+        elif crawl_id_filter:
+            snapshot_scope = snapshot_scope.filter(crawl_id=crawl_id_filter)
+            crawl_scope = crawl_scope.filter(id=crawl_id_filter)
+            archiveresult_scope = archiveresult_scope.filter(snapshot__crawl_id=crawl_id_filter)
 
-        payload = ProgressReport(
-            request,
-            scoped_snapshot=scoped_snapshot,
-            crawl_id_filter=crawl_id_filter,
-            is_admin=is_admin,
-        ).payload()
+        def is_current_run_timestamp(event_ts, run_started_at) -> bool:
+            if run_started_at is None:
+                return True
+            if event_ts is None:
+                return False
+            return event_ts >= run_started_at
+
+        def archiveresult_matches_current_run(ar, run_started_at) -> bool:
+            if run_started_at is None:
+                return True
+            if ar.status in (
+                ArchiveResult.StatusChoices.QUEUED,
+                ArchiveResult.StatusChoices.STARTED,
+                ArchiveResult.StatusChoices.BACKOFF,
+            ):
+                return True
+            event_ts = ar.end_ts or ar.start_ts or ar.modified_at or ar.created_at
+            return is_current_run_timestamp(event_ts, run_started_at)
+
+        def hook_details(hook_name: str, plugin: str = "setup") -> tuple[str, str, str, str]:
+            normalized_hook_name = Path(hook_name).name if hook_name else ""
+            if not normalized_hook_name:
+                return (plugin, plugin, "unknown", "")
+
+            phase = "unknown"
+            if normalized_hook_name == "InstallEvent":
+                phase = "install"
+            elif normalized_hook_name.startswith("on_CrawlSetup__"):
+                phase = "crawl"
+            elif normalized_hook_name.startswith("on_Snapshot__"):
+                phase = "snapshot"
+
+            label = normalized_hook_name
+            if "__" in normalized_hook_name:
+                label = normalized_hook_name.split("__", 1)[1]
+            label = label.rsplit(".", 1)[0]
+            if len(label) > 3 and label[:2].isdigit() and label[2] == "_":
+                label = label[3:]
+            label = label.replace("_", " ").strip() or plugin
+
+            return (plugin, label, phase, normalized_hook_name)
+
+        def process_label(cmd: list[str] | None) -> tuple[str, str, str, str]:
+            hook_path = ""
+            if isinstance(cmd, list) and cmd:
+                hook_path = next(
+                    (arg for arg in cmd if isinstance(arg, str) and Path(arg).name.startswith("on_") and "__" in Path(arg).name),
+                    "",
+                )
+                if not hook_path and isinstance(cmd[0], str):
+                    hook_path = cmd[0]
+
+            if not hook_path:
+                return ("", "setup", "unknown", "")
+
+            return hook_details(Path(hook_path).name, plugin=Path(hook_path).parent.name or "setup")
+
+        def archiveresult_output_path(ar) -> str | None:
+            output_file_map = ar.output_files if isinstance(ar.output_files, dict) else {}
+
+            def is_root_relative(path: str) -> bool:
+                metadata = output_file_map.get(path) or {}
+                return bool(isinstance(metadata, dict) and metadata.get("root_relative"))
+
+            if ar.output_str:
+                raw_output = str(ar.output_str).strip()
+                if ar._looks_like_output_path(raw_output, ar.plugin):
+                    output_path = Path(raw_output)
+                    if output_path.is_absolute():
+                        return None
+
+                    if raw_output.startswith(f"{ar.plugin}/"):
+                        candidates = [raw_output]
+                    elif len(output_path.parts) == 1:
+                        candidates = [f"{ar.plugin}/{raw_output}", raw_output]
+                    else:
+                        candidates = [raw_output]
+
+                    if raw_output in output_file_map and is_root_relative(raw_output):
+                        return raw_output
+
+                    for relative_path in candidates:
+                        plugin_relative = relative_path.removeprefix(f"{ar.plugin}/")
+                        if relative_path in output_file_map:
+                            return f"{ar.plugin}/{relative_path}" if not relative_path.startswith(f"{ar.plugin}/") else relative_path
+                        if plugin_relative in output_file_map:
+                            return f"{ar.plugin}/{plugin_relative}"
+
+            output_file_paths = list(output_file_map.keys())
+            if output_file_paths:
+                fallback_path = ArchiveResult._fallback_output_file_path(output_file_paths, ar.plugin, output_file_map)
+                if fallback_path:
+                    if is_root_relative(fallback_path):
+                        return fallback_path
+                    return f"{ar.plugin}/{fallback_path}"
+
+            return None
+
+        def snapshot_output_url(snapshot, output_path: str) -> str:
+            return build_snapshot_url(str(snapshot["id"]), output_path, request=request, config=request_config)
+
+        def snapshot_archive_path(snapshot) -> str:
+            if snapshot["fs_version"] in ("0.7.0", "0.8.0"):
+                return f"{CONSTANTS.ARCHIVE_DIR_NAME}/{snapshot['timestamp']}"
+            crawl = crawls_by_id.get(str(snapshot["crawl_id"]))
+            username = "web"
+            if crawl is not None and crawl["created_by_id"]:
+                username = crawl["created_by__username"]
+            if username == "system":
+                username = "web"
+            date_base = snapshot["bookmarked_at"] or snapshot["created_at"]
+            date_str = date_base.strftime("%Y%m%d") if date_base else "unknown"
+            domain = Snapshot.extract_domain_from_url(snapshot["url"])
+            return f"{username}/{date_str}/{domain}/{snapshot['id']}"
+
+        def snapshot_view_url(snapshot, output_path: str = "") -> str:
+            anchor = f"#{output_path}" if output_path else ""
+            return build_web_url(
+                f"/{snapshot_archive_path(snapshot)}/index.html{anchor}",
+                request=request,
+                config=request_config,
+            )
+
+        def snapshot_display_url(url: str) -> str:
+            url = str(url or "")
+            return url if len(url) <= 96 else f"{url[:93]}..."
+
+        api_base = get_api_base_url(request=request, config=request_config) if scoped_snapshot is not None else ""
+
+        def screencast_frame_url(crawl_id: str, crawl_dir: Path) -> str:
+            frame_path = crawl_dir / "chrome_screencast" / "latest.jpg"
+            try:
+                frame_stat = frame_path.stat()
+            except OSError:
+                return ""
+            if frame_stat.st_size <= 0:
+                return ""
+            if now.timestamp() - frame_stat.st_mtime > 15:
+                return ""
+            rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}"
+            return f"{api_base}{rel}" if api_base else rel
+
+        machine_id = Machine.current().id
+        orchestrator_proc = (
+            Process.objects.filter(
+                machine_id=machine_id,
+                process_type=Process.TypeChoices.ORCHESTRATOR,
+                status=Process.StatusChoices.RUNNING,
+            )
+            .only("id", "pid", "started_at", "machine_id", "process_type", "status")
+            .order_by("-started_at")
+            .first()
+            if machine_id is not None
+            else None
+        )
+        runner_worker = None
+        orchestrator_proc_running = bool(orchestrator_proc and orchestrator_proc.is_running)
+        if not orchestrator_proc_running:
+            try:
+                from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
+
+                supervisor = get_existing_supervisord_process(quiet=True)
+                runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
+            except (OSError, RuntimeError, TimeoutError):
+                runner_worker = None
+
+        runner_worker_running = bool(runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"))
+        runner_worker_pid = runner_worker.get("pid") if runner_worker else None
+        orchestrator_running = orchestrator_proc_running or runner_worker_running
+        orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else runner_worker_pid
+
+        # Get model counts by status
+        crawl_status_counts = Crawl.status_counts(
+            crawl_scope,
+            (Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.PAUSED),
+        )
+        crawls_queued = crawl_status_counts.get(Crawl.StatusChoices.QUEUED, 0)
+        crawls_active = crawl_status_counts.get(Crawl.StatusChoices.STARTED, 0)
+
+        # Get recent crawls (last 24 hours)
+        from datetime import timedelta
+
+        one_day_ago = now - timedelta(days=1)
+        paused_crawl_cutoff = now - timedelta(hours=12)
+        crawls_recent = crawl_scope.filter(created_at__gte=one_day_ago).count()
+
+        snapshot_status_counts = Snapshot.status_counts(
+            snapshot_scope,
+            Snapshot.OPEN_STATES,
+        )
+        snapshots_queued = snapshot_status_counts.get(Snapshot.StatusChoices.QUEUED, 0)
+        snapshots_active = snapshot_status_counts.get(Snapshot.StatusChoices.STARTED, 0)
+
+        download_plugin_names, indexing_plugin_names = _live_progress_plugin_names()
+        result_statuses = (
+            ArchiveResult.StatusChoices.QUEUED,
+            ArchiveResult.StatusChoices.STARTED,
+        )
+        archiveresult_status_counts = ArchiveResult.status_counts(archiveresult_scope, result_statuses)
+        download_scope = archiveresult_scope.filter(
+            plugin__in=download_plugin_names,
+            snapshot__status__in=Snapshot.RUNNABLE_STATES,
+            snapshot__crawl__status__in=Crawl.RUNNABLE_STATES,
+        )
+        indexing_scope = archiveresult_scope.filter(plugin__in=indexing_plugin_names)
+        download_status_counts = ArchiveResult.status_counts(download_scope, result_statuses)
+        indexing_status_counts = ArchiveResult.status_counts(indexing_scope, result_statuses)
+        archiveresults_queued = archiveresult_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
+        archiveresults_active = archiveresult_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
+
+        downloads_queued = download_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
+        downloads_active = download_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
+        indexing_queued = indexing_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
+        indexing_active = indexing_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
+
+        # Build hierarchical active crawls with nested snapshots and archive results
+        max_active_crawls = 10
+        max_queued_crawls = 10
+        max_started_snapshots_per_crawl = 50
+        max_queued_snapshots_per_crawl = 50
+
+        active_crawl_fields = (
+            "id",
+            "created_at",
+            "created_by_id",
+            "modified_at",
+            "urls",
+            "config",
+            "max_depth",
+            "tags_str",
+            "persona_id",
+            "status",
+            "retry_at",
+            "label",
+            "created_by__id",
+            "created_by__username",
+        )
+        started_crawls = list(
+            crawl_scope.filter(status=Crawl.StatusChoices.STARTED)
+            .values(*active_crawl_fields)
+            .order_by("-modified_at")[:max_active_crawls],
+        )
+        paused_crawls = list(
+            crawl_scope.filter(
+                Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
+                | Q(
+                    status=Crawl.StatusChoices.PAUSED,
+                    snapshot_set__status__in=Snapshot.RUNNABLE_STATES,
+                    snapshot_set__retry_at__lte=now,
+                )
+                | Q(
+                    status=Crawl.StatusChoices.PAUSED,
+                    snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
+                ),
+            )
+            .values(*active_crawl_fields)
+            .distinct()
+            .order_by("-modified_at")[:max_active_crawls],
+        )
+        queued_crawls = list(
+            crawl_scope.filter(status=Crawl.StatusChoices.QUEUED).values(*active_crawl_fields).order_by("-modified_at")[:max_queued_crawls],
+        )
+        queued_crawls_hidden = max(crawls_queued - len(queued_crawls), 0)
+        active_crawls_list = started_crawls + paused_crawls + queued_crawls
+        for crawl in active_crawls_list:
+            crawl["id"] = str(crawl["id"])
+            if crawl["persona_id"]:
+                crawl["persona_id"] = str(crawl["persona_id"])
+        persona_details_by_id: dict[str, dict[str, str]] = {}
+        persona_details_by_name: dict[str, dict[str, str]] = {}
+        persona_objects_by_id = {}
+        persona_objects_by_name = {}
+        persona_ids = {crawl["persona_id"] for crawl in active_crawls_list if crawl["persona_id"]}
+        persona_names = {"Default"} if any(not crawl["persona_id"] for crawl in active_crawls_list) else set()
+        if persona_ids or persona_names:
+            from archivebox.personas.models import Persona
+
+            for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name", "config"):
+                persona_details = {
+                    "name": persona.name,
+                    "admin_url": f"/admin/personas/persona/{persona.pk}/change/",
+                }
+                persona_details_by_id[str(persona.id)] = persona_details
+                persona_details_by_name[persona.name] = persona_details
+                persona_objects_by_id[str(persona.id)] = persona
+                persona_objects_by_name[persona.name] = persona
+        active_crawl_ids = [crawl["id"] for crawl in active_crawls_list]
+        active_crawl_objects = {}
+        if active_crawl_ids:
+            for crawl_obj in Crawl.objects.filter(id__in=active_crawl_ids).select_related("created_by", "persona"):
+                crawl_obj._runtime_config = request_config
+                active_crawl_objects[str(crawl_obj.id)] = crawl_obj
+        snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in active_crawl_ids}
+        cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
+        crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
+        queued_snapshot_overflow_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
+        active_snapshot_scope = snapshot_scope.filter(crawl_id__in=active_crawl_ids)
+        if active_crawl_ids:
+            for row in active_snapshot_scope.values("crawl_id", "status").annotate(count=Count("id")):
+                snapshot_counts_by_crawl.setdefault(str(row["crawl_id"]), {})[row["status"]] = row["count"]
+
+            for row in (
+                active_snapshot_scope.filter(status=Snapshot.StatusChoices.SEALED, downloaded_at__isnull=True)
+                .values("crawl_id")
+                .annotate(count=Count("id"))
+            ):
+                cancelled_snapshot_counts_by_crawl[str(row["crawl_id"])] = row["count"]
+
+            for row in (
+                active_snapshot_scope.filter(
+                    status=Snapshot.StatusChoices.SEALED,
+                )
+                .values("crawl_id")
+                .annotate(size=Sum("output_size"))
+            ):
+                crawl_output_sizes_by_crawl[str(row["crawl_id"])] = int(row["size"] or 0)
+
+        crawl_process_pids: dict[str, int] = {}
+        snapshot_process_pids: dict[str, int] = {}
+        process_records_by_crawl: dict[str, list[tuple[dict[str, object], object | None]]] = {}
+        process_records_by_snapshot: dict[str, list[tuple[dict[str, object], object | None]]] = {}
+        seen_process_records: set[str] = set()
+        crawls_by_id = {str(crawl["id"]): crawl for crawl in active_crawls_list}
+        started_snapshot_fields = (
+            "id_str",
+            "created_at",
+            "modified_at",
+            "url",
+            "timestamp",
+            "bookmarked_at",
+            "crawl_id_str",
+            "title",
+            "downloaded_at",
+            "fs_version",
+            "status",
+        )
+        queued_snapshot_fields = (
+            "id_str",
+            "url",
+            "crawl_id_str",
+            "title",
+            "status",
+        )
+        snapshots = []
+        for crawl_id in active_crawl_ids:
+            crawl_snapshot_scope = active_snapshot_scope.filter(crawl_id=crawl_id)
+            snapshots.extend(
+                crawl_snapshot_scope.filter(status=Snapshot.StatusChoices.STARTED)
+                .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
+                .values(*started_snapshot_fields)
+                .order_by("-modified_at")[:max_started_snapshots_per_crawl],
+            )
+            queued_snapshots = list(
+                crawl_snapshot_scope.filter(status=Snapshot.StatusChoices.QUEUED)
+                .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
+                .values(
+                    *queued_snapshot_fields,
+                )
+                .order_by("modified_at")[:max_queued_snapshots_per_crawl],
+            )
+            queued_snapshot_overflow_by_crawl[str(crawl_id)] = max(
+                snapshot_counts_by_crawl.get(str(crawl_id), {}).get(Snapshot.StatusChoices.QUEUED, 0) - len(queued_snapshots),
+                0,
+            )
+            snapshots.extend(queued_snapshots)
+
+        for snapshot in snapshots:
+            # Process.pwd points at Snapshot.output_dir, which uses CompactUUID
+            # hex path components. Keep progress IDs compact too so process rows
+            # can be matched without carrying dashed/undashed variants.
+            snapshot["id"] = str(snapshot.pop("id_str")).replace("-", "")
+            snapshot["crawl_id"] = str(snapshot.pop("crawl_id_str")).replace("-", "")
+        snapshots_by_id = {str(snapshot["id"]): snapshot for snapshot in snapshots}
+        displayed_snapshots_by_crawl: dict[str, list[Snapshot]] = {str(crawl_id): [] for crawl_id in active_crawl_ids}
+        for snapshot in snapshots:
+            crawl_snapshots = displayed_snapshots_by_crawl.setdefault(str(snapshot["crawl_id"]), [])
+            crawl_snapshots.append(snapshot)
+        displayed_snapshot_ids = [
+            snapshot["id"] for crawl_snapshots in displayed_snapshots_by_crawl.values() for snapshot in crawl_snapshots
+        ]
+        detailed_snapshot_ids = [snapshot["id"] for snapshot in snapshots if snapshot["status"] != Snapshot.StatusChoices.QUEUED]
+        process_value_fields = ("id", "process_type", "status", "pwd", "cmd", "pid", "exit_code", "started_at", "ended_at", "modified_at")
+        if active_crawl_ids or displayed_snapshot_ids:
+            process_scope = Process.objects.filter(
+                machine_id=machine_id,
+                process_type__in=[
+                    Process.TypeChoices.HOOK,
+                    Process.TypeChoices.BINARY,
+                ],
+            )
+            running_processes = process_scope.filter(status=Process.StatusChoices.RUNNING).values(*process_value_fields)
+            recent_processes = (
+                process_scope.filter(modified_at__gte=now - timedelta(minutes=10)).values(*process_value_fields).order_by("-modified_at")
+            )
+        else:
+            running_processes = Process.objects.none()
+            recent_processes = Process.objects.none()
+
+        archiveresults_by_snapshot: dict[str, list[ArchiveResult]] = {str(snapshot_id): [] for snapshot_id in detailed_snapshot_ids}
+        if detailed_snapshot_ids:
+            displayed_archiveresults = (
+                archiveresult_scope.filter(snapshot_id__in=detailed_snapshot_ids)
+                .select_related("process")
+                .only(
+                    "id",
+                    "snapshot_id",
+                    "plugin",
+                    "hook_name",
+                    "status",
+                    "output_str",
+                    "output_files",
+                    "output_size",
+                    "start_ts",
+                    "end_ts",
+                    "created_at",
+                    "modified_at",
+                    "process_id",
+                    "process__id",
+                    "process__pid",
+                    "process__started_at",
+                    "process__timeout",
+                )
+                .order_by("snapshot_id", "start_ts", "created_at")
+            )
+            for archiveresult in displayed_archiveresults:
+                archiveresults_by_snapshot.setdefault(str(archiveresult.snapshot_id), []).append(archiveresult)
+
+        def find_snapshot_for_process(proc_pwd: Path) -> Snapshot | None:
+            for path_part in reversed(proc_pwd.parts):
+                snapshot = snapshots_by_id.get(path_part.replace("-", ""))
+                if snapshot:
+                    return snapshot
+            return None
+
+        def find_crawl_for_process(proc_pwd: Path) -> Crawl | None:
+            for path_part in reversed(proc_pwd.parts):
+                crawl = crawls_by_id.get(path_part.replace("-", ""))
+                if crawl:
+                    return crawl
+            return None
+
+        running_worker_ids: set[str] = set()
+        for proc in running_processes:
+            if not proc["pwd"]:
+                continue
+            proc_pwd = Path(proc["pwd"])
+            matched_snapshot = find_snapshot_for_process(proc_pwd)
+            matched_crawl = (
+                crawls_by_id.get(str(matched_snapshot["crawl_id"])) if matched_snapshot is not None else find_crawl_for_process(proc_pwd)
+            )
+            if matched_snapshot is None:
+                if matched_crawl is None:
+                    continue
+                crawl_id = str(matched_crawl["id"])
+                snapshot_id = ""
+            else:
+                crawl_id = str(matched_snapshot["crawl_id"])
+                snapshot_id = str(matched_snapshot["id"])
+            running_worker_ids.add(str(proc["id"]))
+            _plugin, _label, phase, _hook_name = process_label(proc["cmd"])
+            if crawl_id and proc["pid"]:
+                crawl_process_pids.setdefault(crawl_id, proc["pid"])
+            if phase == "snapshot" and snapshot_id and proc["pid"]:
+                snapshot_process_pids.setdefault(snapshot_id, proc["pid"])
+
+        for proc in recent_processes:
+            if not proc["pwd"]:
+                continue
+            proc_pwd = Path(proc["pwd"])
+            matched_snapshot = find_snapshot_for_process(proc_pwd)
+            matched_crawl = (
+                crawls_by_id.get(str(matched_snapshot["crawl_id"])) if matched_snapshot is not None else find_crawl_for_process(proc_pwd)
+            )
+            if matched_snapshot is None and matched_crawl is None:
+                continue
+            crawl_id = str(matched_snapshot["crawl_id"] if matched_snapshot is not None else matched_crawl["id"])
+            snapshot_id = str(matched_snapshot["id"]) if matched_snapshot is not None else ""
+
+            plugin, label, phase, hook_name = process_label(proc["cmd"])
+
+            record_scope = str(snapshot_id) if phase == "snapshot" and snapshot_id else str(crawl_id)
+            proc_key = f"{record_scope}:{plugin}:{label}:{proc['status']}:{proc['exit_code']}"
+            if proc_key in seen_process_records:
+                continue
+            seen_process_records.add(proc_key)
+
+            status = (
+                "started"
+                if proc["status"] == Process.StatusChoices.RUNNING
+                else (
+                    "skipped"
+                    if proc["exit_code"] == PROCESS_EXIT_SKIPPED or (phase == "binary" and proc["exit_code"] not in (None, 0))
+                    else ("failed" if proc["exit_code"] not in (None, 0) else "succeeded")
+                )
+            )
+            payload: dict[str, object] = {
+                "id": str(proc["id"]),
+                "plugin": plugin,
+                "label": label,
+                "hook_name": hook_name,
+                "status": status,
+                "phase": phase,
+                "source": "process",
+                "process_id": str(proc["id"]),
+            }
+            if status == "started" and proc["pid"]:
+                payload["pid"] = proc["pid"]
+            proc_started_at = proc["started_at"] or proc["modified_at"]
+            proc_run_at = (
+                proc_started_at
+                if proc["status"] == Process.StatusChoices.RUNNING
+                else (proc["ended_at"] or proc["modified_at"] or proc_started_at)
+            )
+            if phase == "snapshot" and snapshot_id:
+                process_records_by_snapshot.setdefault(snapshot_id, []).append((payload, proc_run_at))
+            elif crawl_id:
+                process_records_by_crawl.setdefault(crawl_id, []).append((payload, proc_run_at))
+
+        active_crawls = []
+        total_workers = len(running_worker_ids)
+        for crawl in active_crawls_list:
+            crawl_id = str(crawl["id"])
+            crawl_snapshot_counts = snapshot_counts_by_crawl.get(crawl_id, {})
+            total_snapshots = sum(crawl_snapshot_counts.values())
+            completed_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.SEALED, 0)
+            started_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.STARTED, 0)
+            pending_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.QUEUED, 0)
+            cancelled_snapshots = cancelled_snapshot_counts_by_crawl.get(crawl_id, 0)
+
+            # Count URLs in the crawl (for when snapshots haven't been created yet)
+            urls_count = 0
+            if crawl["urls"]:
+                urls_count = len([u for u in crawl["urls"].split("\n") if u.strip() and not u.startswith("#")])
+
+            # Calculate crawl progress
+            crawl_progress = int((completed_snapshots / total_snapshots) * 100) if total_snapshots > 0 else 0
+            crawl_run_started_at = crawl["created_at"]
+            crawl_setup_plugins = [
+                payload
+                for payload, proc_started_at in process_records_by_crawl.get(crawl_id, [])
+                if is_current_run_timestamp(proc_started_at, crawl_run_started_at)
+            ]
+            crawl_setup_total = len(crawl_setup_plugins)
+            crawl_setup_completed = sum(1 for item in crawl_setup_plugins if item.get("status") == "succeeded")
+            crawl_setup_failed = sum(1 for item in crawl_setup_plugins if item.get("status") == "failed")
+            crawl_setup_pending = sum(1 for item in crawl_setup_plugins if item.get("status") == "queued")
+            crawl_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
+            crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id.replace('-', '')}/change/" if crawl_screencast_url else ""
+
+            # Get active snapshots for this crawl (already prefetched)
+            active_snapshots_for_crawl = []
+            for snapshot in displayed_snapshots_by_crawl.get(crawl_id, []):
+                snapshot_run_started_at = snapshot.get("downloaded_at") or snapshot.get("created_at")
+                snapshot_process_started_at = snapshot.get("created_at")
+                # Get archive results only for displayed active snapshots. Large crawls can
+                # contain thousands of sealed snapshots, and prefetching all their results
+                # makes the progress endpoint compete with the runner.
+                snapshot_results = [
+                    ar
+                    for ar in archiveresults_by_snapshot.get(str(snapshot["id"]), [])
+                    if archiveresult_matches_current_run(ar, snapshot_run_started_at)
+                ]
+                if snapshot["status"] == Snapshot.StatusChoices.QUEUED:
+                    snapshot_results = []
+
+                plugin_progress_values: list[int] = []
+                all_plugins: list[dict[str, object]] = []
+                seen_plugin_keys: set[str] = set()
+                snapshot_title = (
+                    str(snapshot["title"] or "")
+                    if snapshot["status"] == Snapshot.StatusChoices.QUEUED
+                    else Snapshot._normalize_title_candidate(snapshot["title"], snapshot_url=snapshot["url"])
+                )
+                snapshot_favicon_url = ""
+                snapshot_preview_url = ""
+                snapshot_preview_link = ""
+                snapshot_screencast_url = ""
+                snapshot_screencast_link = ""
+                snapshot_fallback_urls: list[str] = []
+                result_by_plugin = {result.plugin: result for result in snapshot_results}
+                title_result = result_by_plugin.get("title")
+                if not snapshot_title and title_result is not None and title_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
+                    snapshot_title = Snapshot._normalize_title_candidate(title_result.output_str, snapshot_url=snapshot["url"])
+                favicon_result = result_by_plugin.get("favicon")
+                if favicon_result is not None and favicon_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
+                    favicon_path = archiveresult_output_path(favicon_result) or "favicon/favicon.ico"
+                    snapshot_favicon_url = snapshot_output_url(snapshot, favicon_path)
+                screenshot_result = result_by_plugin.get("screenshot")
+                if screenshot_result is not None and screenshot_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
+                    snapshot_preview_link = snapshot_view_url(snapshot)
+                    screenshot_path = archiveresult_output_path(screenshot_result) or "screenshot/screenshot.png"
+                    snapshot_preview_url = snapshot_output_url(snapshot, screenshot_path)
+                    snapshot_preview_link = snapshot_view_url(snapshot, screenshot_path)
+                    if snapshot_favicon_url:
+                        snapshot_fallback_urls.append(snapshot_favicon_url)
+                elif snapshot_favicon_url:
+                    snapshot_preview_url = snapshot_favicon_url
+
+                if snapshot["status"] == Snapshot.StatusChoices.STARTED:
+                    snapshot_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
+                    snapshot_screencast_link = snapshot_view_url(snapshot) if snapshot_screencast_url else ""
+
+                def plugin_sort_key(ar):
+                    status_order = {
+                        ArchiveResult.StatusChoices.STARTED: 0,
+                        ArchiveResult.StatusChoices.QUEUED: 1,
+                        ArchiveResult.StatusChoices.SUCCEEDED: 2,
+                        ArchiveResult.StatusChoices.NORESULTS: 3,
+                        ArchiveResult.StatusChoices.FAILED: 4,
+                    }
+                    return (status_order.get(ar.status, 5), ar.plugin, ar.hook_name or "")
+
+                for ar in sorted(snapshot_results, key=plugin_sort_key):
+                    status = ar.status
+                    process = ar.process_record
+                    progress_value = 0
+                    if status in (
+                        ArchiveResult.StatusChoices.SUCCEEDED,
+                        ArchiveResult.StatusChoices.FAILED,
+                        ArchiveResult.StatusChoices.SKIPPED,
+                        ArchiveResult.StatusChoices.NORESULTS,
+                    ):
+                        progress_value = 100
+                    elif status == ArchiveResult.StatusChoices.STARTED:
+                        started_at = ar.start_ts or (process.started_at if process else None)
+                        timeout = process.timeout if process else 120
+                        if started_at and timeout:
+                            elapsed = max(0.0, (now - started_at).total_seconds())
+                            progress_value = int(min(99, max(1, (elapsed / float(timeout)) * 100)))
+                        else:
+                            progress_value = 1
+                    else:
+                        progress_value = 0
+
+                    plugin_progress_values.append(progress_value)
+                    plugin, label, phase, hook_name = hook_details(ar.hook_name or ar.plugin, plugin=ar.plugin)
+
+                    plugin_payload = {
+                        "id": str(ar.id),
+                        "plugin": ar.plugin,
+                        "label": label,
+                        "hook_name": hook_name,
+                        "phase": phase,
+                        "status": status,
+                        "process_id": str(process.id) if process else None,
+                        "admin_url": f"/admin/core/archiveresult/{ar.id}/change/",
+                    }
+                    output_path = archiveresult_output_path(ar)
+                    if output_path:
+                        plugin_payload["output_path"] = output_path
+                        plugin_payload["output_url"] = snapshot_view_url(snapshot, output_path)
+                    if status == ArchiveResult.StatusChoices.STARTED and process:
+                        plugin_payload["pid"] = process.pid
+                    if status == ArchiveResult.StatusChoices.STARTED:
+                        plugin_payload["progress"] = progress_value
+                        plugin_payload["timeout"] = process.timeout if process else 120
+                    plugin_payload["source"] = "archiveresult"
+                    all_plugins.append(plugin_payload)
+                    seen_plugin_keys.add(str(process.id) if process else f"{ar.plugin}:{hook_name}")
+
+                for proc_payload, proc_started_at in process_records_by_snapshot.get(str(snapshot["id"]), []):
+                    if not is_current_run_timestamp(proc_started_at, snapshot_process_started_at):
+                        continue
+                    proc_key = str(proc_payload.get("process_id") or f"{proc_payload.get('plugin')}:{proc_payload.get('hook_name')}")
+                    if proc_key in seen_plugin_keys:
+                        continue
+                    seen_plugin_keys.add(proc_key)
+                    all_plugins.append(proc_payload)
+
+                    proc_status = proc_payload.get("status")
+                    if proc_status in ("succeeded", "failed", "skipped"):
+                        plugin_progress_values.append(100)
+                    elif proc_status == "started":
+                        plugin_progress_values.append(1)
+                    else:
+                        plugin_progress_values.append(0)
+
+                total_plugins = len(all_plugins)
+                completed_plugins = sum(1 for item in all_plugins if item.get("status") == "succeeded")
+                failed_plugins = sum(1 for item in all_plugins if item.get("status") == "failed")
+                pending_plugins = sum(1 for item in all_plugins if item.get("status") == "queued")
+
+                snapshot_progress = int(sum(plugin_progress_values) / len(plugin_progress_values)) if plugin_progress_values else 0
+                worker_state = "running" if snapshot_process_pids.get(str(snapshot["id"])) else "waiting"
+                if (
+                    snapshot["status"] == Snapshot.StatusChoices.STARTED
+                    and worker_state == "waiting"
+                    and not all_plugins
+                    and snapshot["modified_at"]
+                    and (now - snapshot["modified_at"]).total_seconds() > 30
+                ):
+                    worker_state = "waiting" if orchestrator_running else "crashed"
+
+                if snapshot["status"] == Snapshot.StatusChoices.QUEUED and not snapshot_process_pids.get(str(snapshot["id"])):
+                    compact_snapshot = [
+                        str(snapshot["id"]),
+                        snapshot_display_url(snapshot["url"]),
+                    ]
+                    if snapshot_title:
+                        compact_snapshot.append(snapshot_title)
+                    active_snapshots_for_crawl.append(compact_snapshot)
+                    continue
+
+                snapshot_payload = {
+                    "id": str(snapshot["id"]),
+                    "url": snapshot_display_url(snapshot["url"]),
+                    "title": snapshot_title,
+                    "status": snapshot["status"],
+                    "worker_state": worker_state,
+                }
+                if snapshot["status"] != Snapshot.StatusChoices.QUEUED or all_plugins or snapshot_process_pids.get(str(snapshot["id"])):
+                    snapshot_payload.update(
+                        {
+                            "view_url": snapshot_view_url(snapshot),
+                            "started": (snapshot["downloaded_at"] or snapshot["created_at"]).isoformat()
+                            if (snapshot["downloaded_at"] or snapshot["created_at"])
+                            else None,
+                            "progress": snapshot_progress,
+                            "total_plugins": total_plugins,
+                            "completed_plugins": completed_plugins,
+                            "failed_plugins": failed_plugins,
+                            "pending_plugins": pending_plugins,
+                            "all_plugins": all_plugins,
+                        },
+                    )
+                    if snapshot_favicon_url:
+                        snapshot_payload["favicon_url"] = snapshot_favicon_url
+                    if snapshot_preview_url:
+                        snapshot_payload["preview_url"] = snapshot_preview_url
+                        snapshot_payload["preview_link"] = snapshot_preview_link
+                    if snapshot_screencast_url:
+                        snapshot_payload["screencast_url"] = snapshot_screencast_url
+                        snapshot_payload["screencast_link"] = snapshot_screencast_link
+                    if snapshot_fallback_urls:
+                        snapshot_payload["preview_fallbacks"] = snapshot_fallback_urls
+                    if snapshot_process_pids.get(str(snapshot["id"])):
+                        snapshot_payload["worker_pid"] = snapshot_process_pids[str(snapshot["id"])]
+
+                active_snapshots_for_crawl.append(snapshot_payload)
+
+            # Check if crawl can start (for debugging stuck crawls)
+            can_start = bool(crawl["urls"])
+            urls_preview = crawl["urls"][:60] if crawl["urls"] else None
+            crawl_tags = [tag.strip() for tag in (crawl["tags_str"] or "").replace("\n", ",").split(",") if tag.strip()]
+            persona_details = persona_details_by_id.get(str(crawl["persona_id"])) if crawl["persona_id"] else None
+            persona_name = persona_details["name"] if persona_details else "Default"
+            persona_details = persona_details or persona_details_by_name.get(persona_name)
+            crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0)
+            avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0
+            crawl_obj = active_crawl_objects[crawl_id]
+            effective_crawl_config = get_config(crawl=crawl_obj, resolve_plugins=False)
+            max_urls = int(effective_crawl_config.CRAWL_MAX_URLS or 0)
+            crawl_max_size = int(effective_crawl_config.CRAWL_MAX_SIZE or 0)
+            crawl_timeout = int(effective_crawl_config.CRAWL_TIMEOUT or 0)
+            snapshot_max_size = int(effective_crawl_config.SNAPSHOT_MAX_SIZE or 0)
+
+            # Check if retry_at is in the future (would prevent worker from claiming)
+            retry_at_future = crawl["retry_at"] > now if crawl["retry_at"] else False
+            is_paused = crawl_obj.is_paused
+            seconds_until_retry = (
+                0 if is_paused else int((crawl["retry_at"] - now).total_seconds()) if crawl["retry_at"] and retry_at_future else 0
+            )
+            crawl_worker_state = (
+                "running"
+                if crawl_process_pids.get(crawl_id)
+                or any(isinstance(snapshot, dict) and snapshot.get("worker_pid") for snapshot in active_snapshots_for_crawl)
+                else "waiting"
+            )
+            if is_paused:
+                crawl_worker_state = "paused"
+            elif (
+                crawl["status"] == Crawl.StatusChoices.STARTED
+                and crawl_worker_state == "waiting"
+                and (started_snapshots or pending_snapshots)
+            ):
+                crawl_worker_state = "waiting" if orchestrator_running else "crashed"
+
+            active_crawls.append(
+                {
+                    "id": crawl_id,
+                    "label": (next((line.strip() for line in (crawl["urls"] or "").splitlines() if line.strip()), "") or crawl_id)[:60],
+                    "status": crawl["status"],
+                    "is_paused": is_paused,
+                    "started": crawl["created_at"].isoformat() if crawl["created_at"] else None,
+                    "progress": crawl_progress,
+                    "created_by": crawl["created_by__username"],
+                    "persona": persona_name,
+                    "persona_admin_url": persona_details["admin_url"] if persona_details else None,
+                    "max_depth": crawl["max_depth"],
+                    "max_urls": max_urls,
+                    "max_crawl_size": crawl_max_size,
+                    "crawl_timeout": crawl_timeout,
+                    "max_snapshot_size": snapshot_max_size,
+                    "max_crawl_size_display": printable_filesize(crawl_max_size) if crawl_max_size else "unlimited",
+                    "crawl_timeout_display": f"{crawl_timeout}s" if crawl_timeout else "unlimited",
+                    "max_snapshot_size_display": printable_filesize(snapshot_max_size) if snapshot_max_size else "unlimited",
+                    "crawl_output_size": crawl_output_size,
+                    "avg_snapshot_size": avg_snapshot_size,
+                    "crawl_output_size_display": printable_filesize(crawl_output_size) if crawl_output_size else "0 B",
+                    "avg_snapshot_size_display": printable_filesize(avg_snapshot_size) if avg_snapshot_size else "0 B",
+                    "tags": crawl_tags,
+                    "urls_count": urls_count,
+                    "total_snapshots": total_snapshots,
+                    "completed_snapshots": completed_snapshots,
+                    "started_snapshots": started_snapshots,
+                    "failed_snapshots": 0,
+                    "pending_snapshots": pending_snapshots,
+                    "cancelled_snapshots": cancelled_snapshots,
+                    "setup_plugins": crawl_setup_plugins,
+                    "setup_total_plugins": crawl_setup_total,
+                    "setup_completed_plugins": crawl_setup_completed,
+                    "setup_failed_plugins": crawl_setup_failed,
+                    "setup_pending_plugins": crawl_setup_pending,
+                    "screencast_url": crawl_screencast_url,
+                    "screencast_link": crawl_screencast_link,
+                    "active_snapshots": active_snapshots_for_crawl,
+                    "queued_snapshots_hidden": queued_snapshot_overflow_by_crawl.get(crawl_id, 0),
+                    "can_start": can_start,
+                    "urls_preview": urls_preview,
+                    "retry_at_future": retry_at_future,
+                    "seconds_until_retry": seconds_until_retry,
+                    "worker_pid": crawl_process_pids.get(crawl_id),
+                    "worker_state": crawl_worker_state,
+                },
+            )
+
+        payload = {
+            "is_admin": is_admin,
+            "scope": {
+                "snapshot_id": str(scoped_snapshot.id) if scoped_snapshot is not None else "",
+                "crawl_id": crawl_id_filter,
+            },
+            "orchestrator_running": orchestrator_running,
+            "orchestrator_pid": orchestrator_pid,
+            "total_workers": total_workers,
+            "crawls_active": crawls_active,
+            "crawls_queued": crawls_queued,
+            "crawls_recent": crawls_recent,
+            "snapshots_active": snapshots_active,
+            "snapshots_queued": snapshots_queued,
+            "archiveresults_active": archiveresults_active,
+            "archiveresults_queued": archiveresults_queued,
+            "downloads_active": downloads_active,
+            "downloads_queued": downloads_queued,
+            "indexing_active": indexing_active,
+            "indexing_queued": indexing_queued,
+            "active_crawls": active_crawls,
+            "queued_crawls_hidden": queued_crawls_hidden,
+            "server_time": timezone.now().isoformat(),
+        }
+        if is_admin and not snapshot_id_filter and not crawl_id_filter and request.GET.get("collection") == "1":
+            from archivebox.progressmonitor.collection import collection_summary
+
+            payload["collection"] = collection_summary(request.user)
         try:
             import ujson
 

archivebox/search/admin.py

@@ -23,7 +23,7 @@ class SearchResultsChangeList(ChangeList):
         self.search_mode = get_search_mode(request.GET.get("search_mode"), config=request.archivebox_config)
         self.search_mode_backend = get_search_mode_backend(self.search_mode, config=request.archivebox_config)
         super().__init__(request, *args, **kwargs)
-        self.embedded_changelist = request.GET.get("_embedded") in {"crawl", "snapshot"}
+        self.embedded_changelist = request.GET.get("_embedded") == "crawl"
 
     def get_results(self, request):
         """Populate normal admin results plus search-index hint state."""
@@ -69,7 +69,7 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
         if not search_term:
             return super().get_search_results(request, queryset, search_term)
         search_mode = get_search_mode(request.GET.get("search_mode"), config=request.archivebox_config)
-        if queryset.model._meta.label_lower == "core.snapshot" and request.GET.get("_embedded") not in {"crawl", "snapshot"}:
+        if queryset.model._meta.label_lower == "core.snapshot" and request.GET.get("_embedded") != "crawl":
             cached_ids = get_cached_admin_search_ids(request)
             if cached_ids is not None:
                 if not cached_ids:
@@ -84,7 +84,7 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
         if get_search_mode_base(search_mode, config=request.archivebox_config) == "meta":
             qs, use_distinct = super().get_search_results(request, queryset, search_term)
             return qs, use_distinct
-        if request.GET.get("_embedded") in {"crawl", "snapshot"}:
+        if request.GET.get("_embedded") == "crawl":
             try:
                 return queryset.filter(
                     pk__in=query_search_index(

archivebox/search/query.py

@@ -221,6 +221,7 @@ def iter_query_search_ids(
     search_mode: str | None = None,
     config: dict[str, Any] | None = None,
     max_results: int | None = None,
+    stop_event=None,
     **config_kwargs: Any,
 ):
     """Yield snapshot IDs from configured search backend modules."""
@@ -267,6 +268,8 @@ def iter_query_search_ids(
     seen: set[str] = set()
     try:
         for backend_name in backend_names:
+            if stop_event is not None and stop_event.is_set():
+                return
             backend = backends[backend_name]
             try:
                 if backend_name == "sonic":
@@ -292,18 +295,20 @@ def iter_query_search_ids(
                     env=search_backend_command_env(config=config),
                     cwd=CONSTANTS.DATA_DIR,
                     timeout=max(1, int(config.get("TIMEOUT", 60))) * 4,
+                    stop_event=stop_event,
                 )
-                for snapshot_id in ids:
-                    if snapshot_id in seen:
-                        continue
-                    seen.add(snapshot_id)
-                    yield snapshot_id
-                    if max_results and len(seen) >= max_results:
-                        return
+                try:
+                    for snapshot_id in ids:
+                        if snapshot_id in seen:
+                            continue
+                        seen.add(snapshot_id)
+                        yield snapshot_id
+                        if max_results and len(seen) >= max_results:
+                            return
+                finally:
+                    ids.close()
                 successful_backends += 1
             except Exception as err:
-                if isinstance(err, CalledProcessError) and err.stderr:
-                    err.add_note(f"{backend_name} search command stderr:\n{err.stderr.strip()}")
                 errors.append(err)
                 if not fallback_enabled:
                     raise
@@ -315,6 +320,8 @@ def iter_query_search_ids(
             f"[X] The search backend threw an exception={err}:",
             color="red",
         )
+        if isinstance(err, CalledProcessError) and err.stderr:
+            stderr(err.stderr.rstrip(), color="red")
         raise
 
 

archivebox/search/sonic_daemon.py

@@ -16,40 +16,45 @@ def register_sonic_daemon_event_handler(bus) -> None:
         if not isinstance(record, dict) or record.get("type") != "SonicDaemonStartEvent":
             return
 
-        from abx_plugins.plugins.search_backend_sonic.daemon import SonicDaemonStartEvent
+        from abx_plugins.plugins.search_backend_sonic.daemon import (
+            SonicDaemonStartEvent,
+            is_port_listening,
+        )
+        from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
 
-        await asyncio.to_thread(wait_for_sonic_daemon, SonicDaemonStartEvent.from_record(record))
+        daemon_event = SonicDaemonStartEvent.from_record(record)
+        deadline = time.monotonic() + 30.0
+        last_worker = None
 
-    bus.on(ProcessStdoutEvent, on_ProcessStdoutEvent__require_sonic_daemon)
-
-
-def wait_for_sonic_daemon(daemon_event) -> None:
-    """Wait for the Sonic handshake used by both search commands and crawl hooks.
-
-    Supervisord RUNNING only means the process was spawned; its socket can still
-    be opening. Fail immediately for a stopped worker and retain the existing
-    30-second startup deadline for a live worker.
-    """
-    from abx_plugins.plugins.search_backend_sonic.daemon import is_port_listening
-    from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
-
-    deadline = time.monotonic() + 30.0
-    while time.monotonic() < deadline:
-        if is_port_listening(daemon_event.host, daemon_event.port):
-            if time.monotonic() < deadline:
+        while time.monotonic() < deadline:
+            if is_port_listening(daemon_event.host, daemon_event.port):
                 return
-            break
+
+            supervisor = get_existing_supervisord_process(quiet=True)
+            worker = get_worker(supervisor, daemon_event.worker_name) if supervisor is not None else None
+            if isinstance(worker, dict):
+                last_worker = worker
+                if worker.get("statename") not in {"STARTING", "RUNNING", "BACKOFF", "STOPPING"}:
+                    break
+
+            await asyncio.sleep(0.5)
+
+        if is_port_listening(daemon_event.host, daemon_event.port):
+            return
 
         supervisor = get_existing_supervisord_process(quiet=True)
         if supervisor is None:
             raise RuntimeError("Sonic search backend is required, but ArchiveBox supervisord is not running")
+
         worker = get_worker(supervisor, daemon_event.worker_name)
+        if worker is None and last_worker is not None:
+            worker = last_worker
         if not worker:
             raise RuntimeError(f"Sonic search backend worker is not configured: {daemon_event.worker_name}")
-        if worker.get("statename") not in {"STARTING", "RUNNING"}:
+        if worker.get("statename") != "RUNNING":
             raise RuntimeError(
                 f"Sonic search backend worker is {worker.get('statename')}: {worker.get('description')}",
             )
-        time.sleep(min(0.1, max(0.0, deadline - time.monotonic())))
+        raise RuntimeError(f"Sonic search backend is not listening at {daemon_event.url}")
 
-    raise RuntimeError(f"Sonic search backend is not listening at {daemon_event.url}")
+    bus.on(ProcessStdoutEvent, on_ProcessStdoutEvent__require_sonic_daemon)

archivebox/search/views.py

@@ -6,7 +6,7 @@ import json
 import threading
 import time
 from copy import copy
-from queue import Full, Queue
+from queue import Empty, Full, Queue
 from urllib.parse import urlsplit
 from uuid import UUID
 
@@ -56,6 +56,14 @@ def get_cached_admin_search_ids(request) -> list[str] | None:
     return None
 
 
+def get_cached_public_search_ids(request) -> list[str] | None:
+    """Return streamed public search IDs from Django cache."""
+    cached = get_cached_public_search_state(request)
+    if isinstance(cached, dict):
+        return cached.get("ids") or []
+    return None
+
+
 def get_cached_public_search_state(request) -> dict | None:
     """Return streamed public search state from Django cache."""
     cached = cache.get(get_public_search_cache_key(request))
@@ -246,15 +254,18 @@ def iter_filtered_search_result_ids(iterator, queryset, *, flush_max_delay=0.05)
         yield from flush_batch()
 
 
-def iter_search_result_ids(query, base_queryset, *, search_mode, config):
+def iter_search_result_ids(query, base_queryset, *, search_mode, config, stop_event=None):
     """Yield filtered Snapshot IDs from the selected search provider."""
     search_mode_base = get_search_mode_base(search_mode, config=config)
     provider = (
         iter_meta_search_ids(query, base_queryset)
         if search_mode_base == "meta"
-        else iter_query_search_ids(query, search_mode=search_mode, config=config)
+        else iter_query_search_ids(query, search_mode=search_mode, config=config, stop_event=stop_event)
     )
-    yield from iter_filtered_search_result_ids(provider, base_queryset)
+    try:
+        yield from iter_filtered_search_result_ids(provider, base_queryset)
+    finally:
+        provider.close()
 
 
 def snapshot_search_stream_response(query, base_queryset, *, search_mode, config, cache_key, thread_name):
@@ -292,7 +303,7 @@ def snapshot_search_stream_response(query, base_queryset, *, search_mode, config
             iterator = None
             try:
                 close_old_connections()
-                iterator = iter_search_result_ids(query, base_queryset, search_mode=search_mode, config=config)
+                iterator = iter_search_result_ids(query, base_queryset, search_mode=search_mode, config=config, stop_event=stop_event)
                 for snapshot_id in iterator:
                     if stop_event.is_set():
                         break
@@ -318,10 +329,13 @@ def snapshot_search_stream_response(query, base_queryset, *, search_mode, config
                 emit(None)
 
         threading.Thread(target=run_search, name=thread_name, daemon=True).start()
-        yield f"0{stream_padding}\n"
         try:
+            yield f"0{stream_padding}\n"
             while True:
-                item = await asyncio.to_thread(queue.get)
+                try:
+                    item = await asyncio.to_thread(queue.get, True, 0.1)
+                except Empty:
+                    continue
                 if item is None:
                     break
                 if isinstance(item, BaseException):

archivebox/services/crawl_service.py

@@ -10,38 +10,96 @@ from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS
 
 
 class CrawlService(BaseService):
-    """Project crawl phase changes without resurrecting paused or sealed work."""
-
     LISTENS_TO = [CrawlSetupEvent, CrawlStartEvent, CrawlCleanupEvent, CrawlCompletedEvent]
     EMITS = []
 
     def __init__(self, bus, *, crawl_id: str):
         self.crawl_id = crawl_id
         super().__init__(bus)
-        for event_type in self.LISTENS_TO:
-            self.bus.on(event_type, self.on_CrawlEvent__save_to_db)
+        self.bus.on(CrawlSetupEvent, self.on_CrawlSetupEvent__save_to_db)
+        self.bus.on(CrawlStartEvent, self.on_CrawlStartEvent__save_to_db)
+        self.bus.on(CrawlCleanupEvent, self.on_CrawlCleanupEvent__save_to_db)
+        self.bus.on(CrawlCompletedEvent, self.on_CrawlCompletedEvent__save_to_db)
 
-    async def on_CrawlEvent__save_to_db(
-        self,
-        event: CrawlSetupEvent | CrawlStartEvent | CrawlCleanupEvent | CrawlCompletedEvent,
-    ) -> None:
+    async def on_CrawlSetupEvent__save_to_db(self, event: CrawlSetupEvent) -> None:
+        from archivebox.crawls.models import Crawl
+
+        await (
+            Crawl.objects.filter(id=self.crawl_id)
+            .exclude(
+                status__in=Crawl.INACTIVE_STATES,
+            )
+            .aupdate(
+                status=Crawl.StatusChoices.STARTED,
+                retry_at=timezone.now() + timedelta(seconds=ACTIVE_STATE_LEASE_SECONDS),
+                modified_at=timezone.now(),
+            )
+        )
+
+    async def on_CrawlStartEvent__save_to_db(self, event: CrawlStartEvent) -> None:
+        from archivebox.crawls.models import Crawl
+
+        await (
+            Crawl.objects.filter(id=self.crawl_id)
+            .exclude(
+                status__in=Crawl.INACTIVE_STATES,
+            )
+            .aupdate(
+                status=Crawl.StatusChoices.STARTED,
+                retry_at=timezone.now() + timedelta(seconds=ACTIVE_STATE_LEASE_SECONDS),
+                modified_at=timezone.now(),
+            )
+        )
+
+    async def on_CrawlCleanupEvent__save_to_db(self, event: CrawlCleanupEvent) -> None:
+        from archivebox.crawls.models import Crawl
+
+        # Cleanup is still inside the active crawl lifecycle. Snapshot hooks may
+        # have just written discovery output that the runner consumes before the
+        # completion phase, so only CrawlCompleted/finalize_run_state makes the
+        # final sealed-vs-requeue decision.
+        await (
+            Crawl.objects.filter(id=self.crawl_id)
+            .exclude(
+                status__in=Crawl.INACTIVE_STATES,
+            )
+            .aupdate(
+                status=Crawl.StatusChoices.STARTED,
+                retry_at=timezone.now(),
+                modified_at=timezone.now(),
+            )
+        )
+
+    async def on_CrawlCompletedEvent__save_to_db(self, event: CrawlCompletedEvent) -> None:
         from archivebox.crawls.models import Crawl
         from archivebox.core.models import Snapshot
 
-        status = Crawl.StatusChoices.STARTED
-        retry_at = timezone.now()
-        if isinstance(event, (CrawlSetupEvent, CrawlStartEvent)):
-            retry_at += timedelta(seconds=ACTIVE_STATE_LEASE_SECONDS)
-        elif isinstance(event, CrawlCompletedEvent):
-            crawl = await Crawl.objects.aget(id=self.crawl_id)
-            if crawl.status in Crawl.INACTIVE_STATES:
-                return
-            if not await crawl.snapshot_set.filter(status__in=Snapshot.OPEN_STATES).aexists():
-                status, retry_at = Crawl.StatusChoices.SEALED, None
-        # Cleanup remains active: parser outputs may still be projected before
-        # completion. Only completion can seal; every write rechecks cancellation.
+        crawl = await Crawl.objects.aget(id=self.crawl_id)
+        if crawl.is_paused or crawl.status == Crawl.StatusChoices.SEALED:
+            return
+        is_finished = not await crawl.snapshot_set.filter(status__in=Snapshot.OPEN_STATES).aexists()
+        if not is_finished:
+            await (
+                Crawl.objects.filter(id=self.crawl_id)
+                .exclude(
+                    status__in=Crawl.INACTIVE_STATES,
+                )
+                .aupdate(
+                    status=Crawl.StatusChoices.STARTED,
+                    retry_at=timezone.now(),
+                    modified_at=timezone.now(),
+                )
+            )
+            return
+
         await (
             Crawl.objects.filter(id=self.crawl_id)
-            .exclude(status__in=Crawl.INACTIVE_STATES)
-            .aupdate(status=status, retry_at=retry_at, modified_at=timezone.now())
+            .exclude(
+                status__in=Crawl.INACTIVE_STATES,
+            )
+            .aupdate(
+                status=Crawl.StatusChoices.SEALED,
+                retry_at=None,
+                modified_at=timezone.now(),
+            )
         )

archivebox/services/process_service.py

@@ -63,7 +63,8 @@ class ProcessService(BaseService):
 
     def __init__(self, bus):
         self._iface = None
-        self._completed_lock = asyncio.Lock()
+        self._completed_queue: asyncio.Queue[ProcessCompletedEvent | None] = asyncio.Queue()
+        self._completed_worker: asyncio.Task | None = None
         super().__init__(bus)
         self.bus.on(ProcessStartedEvent, self.on_ProcessStartedEvent__save_to_db)
         self.bus.on(ProcessCompletedEvent, self.on_ProcessCompletedEvent__save_to_db)
@@ -75,7 +76,7 @@ class ProcessService(BaseService):
             self._iface = await sync_to_async(current_network_interface_with_machine, thread_sensitive=True)()
         return self._iface
 
-    async def _get_or_create_process(self, event: ProcessStartedEvent | ProcessCompletedEvent):
+    async def on_ProcessStartedEvent__save_to_db(self, event: ProcessStartedEvent) -> None:
         from archivebox.machine.models import Process
 
         iface = await self.current_iface()
@@ -83,7 +84,7 @@ class ProcessService(BaseService):
         worker_type = event.worker_type or ""
         started_at = parse_event_datetime(event.start_ts)
         if started_at is None:
-            raise ValueError(f"{type(event).__name__}.start_ts is required")
+            raise ValueError("ProcessStartedEvent.start_ts is required")
         if event.pid:
             process_query = Process.objects.filter(pid=event.pid, started_at=started_at)
         else:
@@ -112,15 +113,7 @@ class ProcessService(BaseService):
                 status=Process.StatusChoices.RUNNING,
                 retry_at=None,
             )
-        return process, iface, started_at, process_env
-
-    async def on_ProcessStartedEvent__save_to_db(self, event: ProcessStartedEvent) -> None:
-        from archivebox.machine.models import Process
-
-        process, iface, started_at, process_env = await self._get_or_create_process(event)
-        process_type = event.process_type or Process.TypeChoices.HOOK
-        worker_type = event.worker_type or ""
-        if process.iface_id != iface.id or process.machine_id != iface.machine_id:
+        elif process.iface_id != iface.id or process.machine_id != iface.machine_id:
             process.iface = iface
             process.machine = iface.machine
             await process.asave(update_fields=["iface", "machine", "modified_at"])
@@ -172,16 +165,31 @@ class ProcessService(BaseService):
                     process_id=str(process.id),
                 )
 
+    async def _completed_worker_loop(self) -> None:
+        while True:
+            event = await self._completed_queue.get()
+            try:
+                if event is None:
+                    return
+                await self._save_completed_process_to_db(event)
+            finally:
+                self._completed_queue.task_done()
+
+    def _ensure_completed_worker(self) -> None:
+        if self._completed_worker is None or self._completed_worker.done():
+            self._completed_worker = asyncio.create_task(self._completed_worker_loop())
+
     async def on_ProcessCompletedEvent__save_to_db(self, event: ProcessCompletedEvent) -> None:
-        # Serialize completion writes without a detached queue worker. Awaiting
-        # the write here also delivers database failures to the originating event.
-        async with self._completed_lock:
-            await self._save_completed_process_to_db(event)
+        self._ensure_completed_worker()
+        completed_worker = self._completed_worker
+        assert completed_worker is not None
+        await self._completed_queue.put(event)
+        await self.flush_completed()
+        if completed_worker.done():
+            await completed_worker
 
     async def flush_completed(self) -> None:
-        # asyncio locks are FIFO: cleanup waits for earlier completion writers.
-        async with self._completed_lock:
-            pass
+        await self._completed_queue.join()
 
     async def on_CrawlCleanupEvent__flush_completed(self, event: CrawlCleanupEvent) -> None:
         await self.flush_completed()
@@ -192,9 +200,43 @@ class ProcessService(BaseService):
     async def _save_completed_process_to_db(self, event: ProcessCompletedEvent) -> None:
         from archivebox.machine.models import Process
 
-        process, iface, started_at, process_env = await self._get_or_create_process(event)
+        iface = await self.current_iface()
         process_type = event.process_type or Process.TypeChoices.HOOK
         worker_type = event.worker_type or ""
+        started_at = parse_event_datetime(event.start_ts)
+        if started_at is None:
+            raise ValueError("ProcessCompletedEvent.start_ts is required")
+        if event.pid:
+            process_query = Process.objects.filter(pid=event.pid, started_at=started_at)
+        else:
+            process_query = Process.objects.filter(
+                process_type=process_type,
+                worker_type=worker_type,
+                pwd=event.output_dir,
+                started_at=started_at,
+            )
+        process = await process_query.order_by("-modified_at").afirst()
+        process_env = normalize_process_env(event.env)
+        if process is None:
+            await Process.objects.acreate(
+                machine=iface.machine,
+                iface=iface,
+                parent_id=None,
+                process_type=process_type,
+                worker_type=worker_type,
+                pwd=event.output_dir,
+                cmd=[event.hook_path, *event.hook_args],
+                env=process_env,
+                timeout=event.timeout,
+                pid=event.pid or None,
+                url=event.url or None,
+                started_at=started_at,
+                status=Process.StatusChoices.RUNNING,
+                retry_at=None,
+            )
+            process = await process_query.order_by("-modified_at").afirst()
+            if process is None:
+                return
 
         missing_cmd = not process.cmd
         updates = {

archivebox/services/resource_admission.py

@@ -0,0 +1,91 @@
+"""Admission signals, not estimates of how much RAM an extractor will need."""
+
+import math
+import os
+import time
+from pathlib import Path
+
+
+RESOURCE_RECHECK_SECONDS = 2.0
+# Scheduling policy, not a minimum-RAM claim: stop increasing extraction
+# load when all non-idle work loses at least one second out of ten to memory
+# stalls. Require both recent and rolling pressure so transient reclaim does
+# not indefinitely starve otherwise viable work on small hosts.
+MEMORY_FULL_STALL_PERCENT = 10.0
+
+
+class ResourceAdmission:
+    def __init__(self):
+        self._memory_stalls: dict[str, tuple[int, float]] = {}
+        self._checked_at = float("-inf")
+        self._reason: str | None = None
+
+    @staticmethod
+    def cgroup_paths() -> list[Path]:
+        root = Path("/sys/fs/cgroup")
+        paths = [root]
+        try:
+            for line in Path("/proc/self/cgroup").read_text().splitlines():
+                if line.startswith("0::"):
+                    current = root / line[3:].lstrip("/")
+                    if current.is_dir() and current.is_relative_to(root):
+                        paths = [current, *[parent for parent in current.parents if parent.is_relative_to(root)]]
+                    break
+        except OSError:
+            pass
+        return paths
+
+    def snapshot_limit(self, configured: int) -> int:
+        cpus = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 1)
+        for path in self.cgroup_paths():
+            try:
+                quota, period = (path / "cpu.max").read_text().split()
+                if quota != "max":
+                    cpus = min(cpus, max(1, math.ceil(int(quota) / int(period))))
+            except (OSError, ValueError, ZeroDivisionError):
+                continue
+        # CPU quota limits parallel admission, not forward progress. A host
+        # with a fractional CPU must still be able to run one snapshot.
+        return max(1, min(configured, cpus))
+
+    def observe_memory_stalls(self, source: str, total: int, avg10: float, sampled_at: float) -> bool:
+        previous = self._memory_stalls.get(source)
+        self._memory_stalls[source] = (total, sampled_at)
+        if previous is None:
+            return avg10 >= MEMORY_FULL_STALL_PERCENT
+        previous_total, previous_time = previous
+        elapsed = sampled_at - previous_time
+        if elapsed <= 0 or total < previous_total:
+            return False
+        stalled_percent = (total - previous_total) / (elapsed * 1_000_000) * 100
+        return avg10 >= MEMORY_FULL_STALL_PERCENT and stalled_percent >= MEMORY_FULL_STALL_PERCENT
+
+    def memory_pressure(self) -> str | None:
+        now = time.monotonic()
+        if now - self._checked_at < RESOURCE_RECHECK_SECONDS:
+            return self._reason
+        self._checked_at = now
+        reasons = []
+        cgroups = self.cgroup_paths()
+        for path in cgroups:
+            try:
+                current = int((path / "memory.current").read_text())
+                for name in ("memory.high", "memory.max"):
+                    limit = (path / name).read_text().strip()
+                    if limit != "max" and current >= int(limit):
+                        reasons.append(f"{name} reached")
+            except (OSError, ValueError):
+                continue
+        for path in [Path("/proc/pressure/memory"), *[p / "memory.pressure" for p in cgroups]]:
+            try:
+                full = next(line for line in path.read_text().splitlines() if line.startswith("full "))
+                fields = dict(field.split("=", 1) for field in full.split()[1:])
+                if self.observe_memory_stalls(str(path), int(fields["total"]), float(fields["avg10"]), now):
+                    reasons.append("memory stalls")
+            except (OSError, ValueError, KeyError, StopIteration):
+                continue
+        self._reason = ", ".join(dict.fromkeys(reasons)) or None
+        return self._reason
+
+
+resource_admission = ResourceAdmission()

archivebox/services/runner/__init__.py

@@ -1,16 +0,0 @@
-"""ArchiveBox scheduling and execution entry points.
-
-scheduler selects due work; dispatch claims it; crawl runs hook events and
-projects their results; install runs dependency-only event buses.
-"""
-
-from .crawl import CrawlRunner as CrawlRunner
-from .scheduler import ensure_background_runner as ensure_background_runner
-from .crawl import run_crawl as run_crawl
-from .install import run_binary as run_binary
-from .install import run_install as run_install
-from .maintenance import run_snapshot_maintenance as run_snapshot_maintenance
-from .dispatch import run_due_crawl as run_due_crawl
-from .dispatch import run_due_snapshot as run_due_snapshot
-from .dispatch import run_due_binary as run_due_binary
-from .scheduler import run_pending_crawls as run_pending_crawls

archivebox/services/runner/console.py

@@ -1,82 +0,0 @@
-from __future__ import annotations
-import os
-import sys
-import shutil
-from rich.console import Console
-from rich.text import Text
-from abx_dl.catalog import PluginCatalog
-
-
-def _runner_short_id(identifier) -> str:
-    return str(identifier).replace("-", "")[-8:]
-
-
-def _runner_label(value: str, *, reserve: int) -> str:
-    width = max(24, shutil.get_terminal_size(fallback=(120, 40)).columns - reserve)
-    value = " ".join(str(value or "").split())
-    if len(value) <= width:
-        return value
-    return f"{value[: max(0, width - 3)]}..."
-
-
-def _runner_console_line(*, crawl=None, crawl_id=None, snapshot=None, status: str = "STARTED") -> None:
-    crawl_id = crawl.id if crawl is not None else crawl_id
-    line = Text()
-    line.append(f"[Crawl#{_runner_short_id(crawl_id)}]", style="cyan bold")
-    line.append(" ")
-    if snapshot is not None:
-        line.append(f"[Snapshot#{_runner_short_id(snapshot.id)}]", style="magenta bold")
-        line.append(" ")
-    status_styles = {
-        "STARTED": "green bold",
-        "SEALED": "blue bold",
-        "PAUSED": "yellow bold",
-    }
-    line.append(f"[{status}]", style=status_styles.get(status, "white bold"))
-    line.append(" ")
-    prefix_width = len(line.plain)
-    if snapshot is not None:
-        label = snapshot.url
-    else:
-        label = (crawl.label or "").strip()
-        if not label:
-            label = (crawl.urls or "").partition("\n")[0].strip() or str(crawl_id)
-    line.append(_runner_label(label, reserve=prefix_width))
-    Console(highlight=False).print(line)
-
-
-def _count_selected_hooks(catalog: PluginCatalog, selected_plugins: list[str] | None) -> int:
-    selected = catalog.select(selected_plugins) if selected_plugins else catalog
-    return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name)
-
-
-def create_console():
-    """Build a correctly sized progress console and return its owned tty stream.
-
-    Callers close only the returned stream; stdout/stderr belong to the process.
-    A crawl may use this console noninteractively, while installation displays
-    the live UI only when a terminal is attached.
-    """
-    stdout_is_tty = sys.stdout.isatty()
-    stderr_is_tty = sys.stderr.isatty()
-    interactive = stdout_is_tty or stderr_is_tty
-    stream = sys.stderr if stderr_is_tty or not stdout_is_tty else sys.stdout
-    owned_stream = None
-    if interactive and os.path.exists("/dev/tty"):
-        try:
-            owned_stream = open("/dev/tty", "w", buffering=1, encoding=stream.encoding or "utf-8")
-            stream = owned_stream
-        except OSError:
-            pass
-    try:
-        size = os.get_terminal_size(stream.fileno())
-    except (AttributeError, OSError, ValueError):
-        size = shutil.get_terminal_size(fallback=(160, 40))
-    console = Console(
-        file=stream,
-        force_terminal=interactive,
-        width=size.columns,
-        height=size.lines,
-        _environ={"COLUMNS": str(size.columns), "LINES": str(size.lines)},
-    )
-    return console, owned_stream, interactive