release: v0.9.35rc67

Clarify security risks and update information on CSRF/XSS protections.

.claude/settings.json

@@ -1,5 +0,0 @@
-{
-  "enabledPlugins": {
-    "pyright-lsp@claude-plugins-official": true
-  }
-}

.claude/settings.local.json

@@ -1,51 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Read(**)",
-      "Glob(**)",
-      "Grep(**)",
-      "Bash(python -m archivebox:*)",
-      "Bash(ls:*)",
-      "Bash(xargs:*)",
-      "Bash(python -c:*)",
-      "Bash(printf:*)",
-      "Bash(pkill:*)",
-      "Bash(python3:*)",
-      "Bash(sqlite3:*)",
-      "WebFetch(domain:github.com)",
-      "Bash(uv add:*)",
-      "Bash(mkdir:*)",
-      "Bash(chmod:*)",
-      "Bash(python -m forum_dl:*)",
-      "Bash(archivebox manage migrate:*)",
-      "Bash(cat:*)",
-      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
-      "Bash(forum-dl:*)",
-      "Bash(pip uninstall:*)",
-      "Bash(python:*)",
-      "Bash(source .venv/bin/activate)",
-      "Bash(mv:*)",
-      "Bash(echo:*)",
-      "Bash(grep:*)",
-      "WebFetch(domain:python-statemachine.readthedocs.io)",
-      "Bash(./bin/run_plugin_tests.sh:*)",
-      "Bash(done)",
-      "Bash(coverage erase:*)",
-      "Bash(gh api:*)"
-    ]
-  },
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Bash",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
-            "statusMessage": "Checking working directory..."
-          }
-        ]
-      }
-    ]
-  }
-}

.dockerignore

@@ -3,18 +3,33 @@
 /.heartbeat.json
 /package.json
 /package-lock.json
+__pycache__
+**/__pycache__
 *.pyc
-__pycache__/
+*.pyo
+*.py[cod]
+*$py.class
 .mypy_cache/
 .pytest_cache/
+.ruff_cache/
+.uv-cache/
+.cache/
+.cache*/
 .github/
 .pdm-build/
+.pdm-build/**
 .pdm-python
 .eggs/
+*.egg-info/
 .git/
+!.git/
+.git/*
 .vscode/
 !.git/HEAD
-!.git/refs/heads/*
+!.git/packed-refs
+!.git/refs/
+!.git/refs/heads/
+!.git/refs/heads/**
 
 venv/
 .venv/
@@ -31,6 +46,7 @@ chromeprofile/
 chrome_profile/
 lib/
 out/
+tests/out/
 users/
 archive/
 crawls/
@@ -38,6 +54,8 @@ snapshots/
 logs/
 archivebox-docker-smoke*/
 archivebox-compose-smoke*/
+docker-test/
+docker-test*/
 core
 *.core
 
@@ -50,6 +68,8 @@ dist/
 brew_dist/
 deb_dist/
 pip_dist/
+cache/
+caches/
 assets/
 docker/
 website/

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -69,7 +69,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:

.github/ISSUE_TEMPLATE/2-feature_request.yml

@@ -71,7 +71,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:

.github/SECURITY.md

@@ -2,18 +2,15 @@
 
 ---
 
-## Security Information
+## Security Documentation
 
-Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
-
-https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
-
-Also see this section of the README about important caveats when running ArchiveBox:
-
-https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
-
-You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
+Please start by reading these links to understand our existing security model, known issues, API surfaces, and setup steps:
 
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+- https://github.com/ArchiveBox/ArchiveBox/security/advisories/
+- https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication
 - https://github.com/ArchiveBox/ArchiveBox#archive-layout
 - https://github.com/ArchiveBox/ArchiveBox#archivebox-development
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
@@ -32,3 +29,43 @@ We use Github's built-in [Private Reporting](https://docs.github.com/en/code-sec
 3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers
 
 You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).
+
+---
+
+## CVEs and Report Credits
+
+> [!IMPORTANT]
+> **Reports that only affect `dev` or beta versions will be converted to normal Github issues and will not be issued CVEs.**  
+> Reporters **will still receive credit in release notes** however.
+
+> [!TIP]
+> Reports that affect **stable published versions will be issued CVEs** and published w/ full credit.
+
+Please read our existing published [security advisories](https://github.com/ArchiveBox/ArchiveBox/security/advisories), [Security Overview Docs](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview), and [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and avoid creating duplicates.
+
+---
+
+## Threat Model
+
+### All admin users are assumed to be trusted
+
+ArchiveBox does not support granular user permissions, there are only two types of users available in the Admin UI & REST API:
+
+- logged in admin superusers (which are assumed to have `root` on the ArchiveBox host machine already)
+- non-logged in anonymous guest users
+
+Because admin / REST API / CLI / Python API users are already assumed to have `root` on the host machine, there are deliberately no protections against admins inputting malicious config values, attempts to attack other admin users, or destructive/malicious actions. Permissions for non-logged in users are managed via the [`PERMISSIONS`](https://github.com/archiveBox/archiveBox/wiki/configuration#permissions) + [`PUBLIC_INDEX` + `PUBLIC_ADD_VIEW`](https://github.com/archiveBox/archiveBox/wiki/configuration#public_index--public_add_view) + other config options.
+
+### Sanitization of Config Values, Filesystem Paths, DB Fields, etc.
+
+Non-logged in users can only choose between public Persona config presets on the `/add/` page, but they should never be allowed to directly set config options like `*_ARGS`, `*_BINARY` or DB fields like `url`, `title`, `tag.name`, `notes` without sanitization/filtering on save and/or use.
+
+Non-logged in users should be tightly limited to prevent attacks like SQL/shell injection, path traversal attacks, XSS, cookie leaks between admin/non-admin domains, etc. Any reports related to non-logged-in users will be prioritized as this is a surface we care a lot about hardening.
+
+### XSS, CSRF, CORS, CSP
+
+`< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
+
+[`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
+
+Note archivebox is designed to be able to archive any URL the server can reach, including internal intranet URLs. The only URLs that are blocked are URLs hosted by the archivebox server itself.

.github/scripts/clone_abx_repo.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+repo_name="$1"
+target_dir="${2:-$repo_name}"
+
+case "$repo_name" in
+    abxpkg)
+        version="1.11.230"
+        ;;
+    abx-plugins)
+        version="1.11.238"
+        ;;
+    *)
+version="$(
+python3 - "$repo_name" <<'PY'
+import re
+import sys
+from pathlib import Path
+
+repo_name = sys.argv[1]
+lock_text = Path("uv.lock").read_text()
+match = re.search(
+    rf'^\[\[package\]\]\s*\nname = "{re.escape(repo_name)}"\s*\nversion = "([^"]+)"',
+    lock_text,
+    re.MULTILINE,
+)
+if not match:
+    raise SystemExit(f"Could not find {repo_name} in uv.lock")
+print(match.group(1))
+PY
+)"
+        ;;
+esac
+
+echo "Cloning ArchiveBox/${repo_name}@v${version} into ${target_dir}"
+git clone --depth=1 --branch "v${version}" "https://github.com/ArchiveBox/${repo_name}.git" "$target_dir"

.github/workflows/debian.yml

@@ -1,214 +0,0 @@
-name: Build Debian package
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'pkg/debian/**'
-      - 'bin/build_deb.sh'
-      - 'bin/release_deb.sh'
-      - '.github/workflows/debian.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        arch: [amd64, arm64]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Install nfpm
-        env:
-          NFPM_VERSION: "2.45.1"
-        run: |
-          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \
-            | sudo tar -xz -C /usr/local/bin nfpm
-          nfpm --version
-
-      - name: Build .deb package
-        run: |
-          export VERSION="${{ steps.version.outputs.version }}"
-          export ARCH="${{ matrix.arch }}"
-          ./bin/build_deb.sh
-
-      - name: Verify .deb package contents
-        run: |
-          DEB_FILE="$(ls dist/archivebox*.deb | head -1)"
-          echo "=== Package info ==="
-          dpkg-deb --info "$DEB_FILE"
-          echo ""
-          echo "=== Package contents ==="
-          dpkg-deb --contents "$DEB_FILE"
-          echo ""
-          echo "=== Control fields ==="
-          dpkg-deb --field "$DEB_FILE"
-
-      - name: Upload .deb artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb
-          path: dist/*.deb
-
-  test:
-    needs: build
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-            runner: ubuntu-24.04
-          - arch: arm64
-            runner: ubuntu-24.04-arm
-    runs-on: ${{ matrix.runner }}
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local wheel
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --wheel --out-dir /tmp/wheels/
-
-      - name: Download .deb artifact
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*-${{ matrix.arch }}.deb
-          merge-multiple: true
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget
-
-      - name: Pre-seed virtualenv with local wheel before dpkg install
-        run: |
-          # CI-only: pre-seed the venv with a local wheel so we test the
-          # unreleased code, not whatever is on PyPI. We explicitly use python3.13
-          # here (matching the system dep installed above) to ensure the venv is
-          # created with the correct Python version. On real installs, install.sh
-          # handles this by preferring python3.13 and failing if < 3.13.
-          # When postinstall.sh runs during dpkg -i, it finds the venv already
-          # populated and just upgrades deps.
-          sudo mkdir -p /opt/archivebox
-          sudo python3.13 -m venv /opt/archivebox/venv
-          sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools
-          sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl
-          echo "[√] Pre-seeded /opt/archivebox/venv with local wheel"
-
-      - name: Install .deb package
-        run: |
-          # dpkg install will run postinstall which creates the user,
-          # sets up systemd, and tries pip install (which finds it already installed)
-          sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y
-
-      - name: Verify archivebox is installed
-        run: |
-          which archivebox
-          archivebox version
-
-      - name: Verify wrapper script works
-        run: |
-          /usr/bin/archivebox version
-          /usr/bin/archivebox --help | head -5
-
-      - name: Test archivebox init as archivebox user
-        run: |
-          # The postinstall should have created the user
-          id archivebox
-          sudo mkdir -p /tmp/archivebox-test
-          sudo chown archivebox:archivebox /tmp/archivebox-test
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init'
-
-      - name: Test archivebox status
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status'
-
-      - name: Test archivebox add
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"'
-
-      - name: Verify systemd service file exists
-        run: |
-          test -f /usr/lib/systemd/system/archivebox.service
-          cat /usr/lib/systemd/system/archivebox.service
-
-  # Upload .deb to GitHub Release only after tests pass
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: [build, test]
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: write
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Determine release tag
-        id: tag
-        run: |
-          TAG="${{ github.event.release.tag_name }}"
-          if [ -z "$TAG" ]; then
-            TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')"
-            echo "[i] No release tag in event context, using version from pyproject.toml: $TAG"
-          fi
-          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
-
-      - name: Download all .deb artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*.deb
-          merge-multiple: true
-
-      - name: Upload .deb to GitHub Release
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          TAG="${{ steps.tag.outputs.tag }}"
-          # Verify the release exists before uploading
-          if ! gh release view "$TAG" >/dev/null 2>&1; then
-            echo "[!] No GitHub Release found for tag $TAG."
-            if [ -n "${{ github.event.release.tag_name }}" ]; then
-              echo "[X] This was triggered by a release event — the release should exist. Failing."
-              exit 1
-            fi
-            echo "[i] Skipping upload (workflow_dispatch without a release)."
-            echo "    Create a release first or trigger via the release event."
-            exit 0
-          fi
-          gh release upload "$TAG" *.deb --clobber

.github/workflows/docker.yml

@@ -5,7 +5,7 @@ on:
   workflow_call:
   push:
     branches:
-      - dev
+      - "**"
     tags:
       - 'v*'
   # pull_request:
@@ -19,7 +19,7 @@ permissions:
   packages: write
 
 concurrency:
-  group: docker-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -45,6 +45,47 @@ jobs:
         #   submodules: true
         #   fetch-depth: 1
 
+      - name: Wait for released ArchiveBox deps on PyPI
+        run: |
+          python3 - <<'PY'
+          import json
+          import re
+          import sys
+          import time
+          import tomllib
+          import urllib.request
+
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  try:
+                      with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20):
+                          available = True
+                  except Exception:
+                      available = False
+                  if available:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
+          PY
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
@@ -58,6 +99,41 @@ jobs:
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
+      - name: Wait for published abx-dl image
+        id: abx_dl_image
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          ABX_DL_VERSION="$(python3 - <<'PY'
+          import re
+          import tomllib
+
+          with open("pyproject.toml", "rb") as f:
+              deps = tomllib.load(f)["project"]["dependencies"]
+
+          for dep in deps:
+              match = re.match(r"abx-dl\s*(?:==|>=)\s*([^,;\s]+)", dep)
+              if match:
+                  print(match.group(1))
+                  break
+          else:
+              raise SystemExit("Missing abx-dl dependency in pyproject.toml")
+          PY
+          )"
+          ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
+          deadline=$((SECONDS + 1800))
+          until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
+            if (( SECONDS >= deadline )); then
+              echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2
+              exit 1
+            fi
+            echo "${ABX_DL_IMAGE} is not published yet; waiting..."
+            sleep 30
+          done
+
+          echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
+          docker buildx imagetools inspect "${ABX_DL_IMAGE}"
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         if: github.event_name != 'pull_request'
@@ -78,13 +154,10 @@ jobs:
         run: |
           set -Eeuo pipefail
           VERSION="$(python3 - <<'PY'
-          from pathlib import Path
-          import re
+          import tomllib
 
-          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
-          if not match:
-              raise SystemExit("Failed to read version from pyproject.toml")
-          print(match.group(1))
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
           PY
           )"
 
@@ -93,11 +166,12 @@ jobs:
             echo "org.opencontainers.image.version=${VERSION}"
             echo "org.opencontainers.image.revision=${GITHUB_SHA}"
             echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
+            echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
             echo 'EOF'
             echo "version=${VERSION}"
           } >> "$GITHUB_OUTPUT"
 
-          echo "[+] Building ${{ matrix.platform }} for ${VERSION}"
+          echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
 
       - name: Build and push digest
         id: docker_build
@@ -111,14 +185,48 @@ jobs:
             ${{ env.DOCKERHUB_IMAGE }}
             ${{ env.GHCR_IMAGE }}
           labels: ${{ steps.docker_meta.outputs.labels }}
+          build-args: |
+            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
+            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
           cache-from: type=gha,scope=${{ matrix.cache_scope }}
           cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
+          pull: true
           platforms: ${{ matrix.platform }}
           outputs: type=image,push-by-digest=true,name-canonical=true,push=true
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
 
+      - name: Validate pushed image version, commit, and size
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          IMAGE="${DOCKERHUB_IMAGE}@${{ steps.docker_build.outputs.digest }}"
+          SHORT_SHA="${GITHUB_SHA::7}"
+
+          DATA_DIR="$(mktemp -d)"
+          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
+          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version | tee /tmp/archivebox-version.out
+          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
+          docker run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt > /tmp/archivebox-image-version.txt
+          grep -F "ArchiveBox v${{ steps.docker_meta.outputs.version }}" /tmp/archivebox-version.out
+          grep -F "COMMIT_HASH=${SHORT_SHA}" /tmp/archivebox-version.out
+          grep -Fx "COMMIT_HASH=${GITHUB_SHA}" /tmp/archivebox-image-version.txt
+
+          python3 - "$IMAGE" <<'PY'
+          import json
+          import subprocess
+          import sys
+
+          image = sys.argv[1]
+          limit = 780 * 1024 * 1024
+          manifest = json.loads(subprocess.check_output(["docker", "manifest", "inspect", image]))
+          total = manifest.get("config", {}).get("size", 0) + sum(layer.get("size", 0) for layer in manifest.get("layers", []))
+          print(f"{image} compressed_size={total / 1024 / 1024:.2f} MiB limit={limit / 1024 / 1024:.2f} MiB")
+          if total > limit:
+              raise SystemExit(f"{image} is over the compressed size limit")
+          PY
+
       - name: Export digest
         shell: bash
         run: |
@@ -177,36 +285,49 @@ jobs:
         run: |
           set -Eeuo pipefail
           VERSION="$(python3 - <<'PY'
-          from pathlib import Path
-          import re
+          import tomllib
 
-          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
-          if not match:
-              raise SystemExit("Failed to read version from pyproject.toml")
-          print(match.group(1))
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
           PY
           )"
-          SHORT_SHA="${GITHUB_SHA::8}"
+          BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)"
+          SHORT_SHA="${GITHUB_SHA::12}"
+          test -n "$BRANCH_TAG"
+          test -n "$SHORT_SHA"
 
           {
             echo 'dockerhub_tags<<EOF'
-            echo "${DOCKERHUB_IMAGE}:dev"
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${DOCKERHUB_IMAGE}:latest"
+            fi
+            echo "${DOCKERHUB_IMAGE}:${BRANCH_TAG}"
             echo "${DOCKERHUB_IMAGE}:${VERSION}"
             echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
             echo 'EOF'
             echo 'ghcr_tags<<EOF'
-            echo "${GHCR_IMAGE}:dev"
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${GHCR_IMAGE}:latest"
+            fi
+            echo "${GHCR_IMAGE}:${BRANCH_TAG}"
             echo "${GHCR_IMAGE}:${VERSION}"
             echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
             echo 'EOF'
             echo "version=${VERSION}"
+            echo "branch_tag=${BRANCH_TAG}"
             echo "short_sha=${SHORT_SHA}"
           } >> "$GITHUB_OUTPUT"
 
           echo "[+] Publishing Docker Hub tags:"
-          printf '%s\n' "${DOCKERHUB_IMAGE}:dev" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${DOCKERHUB_IMAGE}:latest"
+          fi
+          printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
           echo "[+] Publishing GHCR tags:"
-          printf '%s\n' "${GHCR_IMAGE}:dev" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${GHCR_IMAGE}:latest"
+          fi
+          printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
 
       - name: Create Docker Hub manifest
         shell: bash
@@ -250,8 +371,12 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
-          docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
-          docker buildx imagetools inspect "${GHCR_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
+          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
+          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
 
       - name: Update README
         uses: peter-evans/dockerhub-description@v4

.github/workflows/homebrew.yml

@@ -1,233 +0,0 @@
-name: Build Homebrew formula
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'brew_dist/**'
-      - 'bin/build_brew.sh'
-      - 'bin/release_brew.sh'
-      - '.github/workflows/homebrew.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-and-test:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-latest, ubuntu-24.04]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Validate formula template syntax
-        run: ruby -c brew_dist/archivebox.rb
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies (Linux)
-        if: runner.os == 'Linux'
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local sdist
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --sdist --out-dir /tmp/sdist/
-
-      - name: Generate formula from local sdist
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)"
-          SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')"
-
-          # Install archivebox + poet into a temp venv to generate resource stanzas
-          python3 -m venv /tmp/poet-venv
-          source /tmp/poet-venv/bin/activate
-          pip install --quiet "$SDIST_PATH" homebrew-pypi-poet
-          echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
-          RESOURCES="$(poet archivebox)"
-          deactivate
-
-          # For CI: use file:// URL pointing to local sdist
-          # For release: this gets overridden with the PyPI URL
-          SDIST_URL="file://${SDIST_PATH}"
-
-          cat > /tmp/archivebox.rb << RUBY
-class Archivebox < Formula
-  include Language::Python::Virtualenv
-
-  desc "Self-hosted internet archiving solution"
-  homepage "https://github.com/ArchiveBox/ArchiveBox"
-  url "${SDIST_URL}"
-  sha256 "${SDIST_SHA256}"
-  license "MIT"
-
-  depends_on "python@3.13"
-
-${RESOURCES}
-
-  def install
-    virtualenv_install_with_resources
-  end
-
-  def post_install
-    data_dir = var/"archivebox"
-    data_dir.mkpath
-    ENV["DATA_DIR"] = data_dir.to_s
-    system bin/"archivebox", "init"
-  end
-
-  def caveats
-    <<~EOS
-      ArchiveBox data is stored in:
-        #{var}/archivebox
-
-      To start archiving, run:
-        cd #{var}/archivebox && archivebox add 'https://example.com'
-
-      To start the web UI:
-        cd #{var}/archivebox && archivebox server 0.0.0.0:8000
-    EOS
-  end
-
-  test do
-    assert_match version.to_s, shell_output("#{bin}/archivebox version")
-  end
-end
-RUBY
-
-          echo "[√] Generated formula:"
-          ruby -c /tmp/archivebox.rb
-          cat /tmp/archivebox.rb
-
-      - name: Install Homebrew (Linux only)
-        if: runner.os == 'Linux'
-        run: |
-          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
-          echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV"
-          echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV"
-          echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV"
-          echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH"
-          echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH"
-
-      - name: Install brew dependencies
-        run: brew install python@3.13
-
-      - name: Install archivebox via brew from local formula
-        run: |
-          brew install --build-from-source --verbose /tmp/archivebox.rb
-
-      - name: Verify archivebox version
-        run: archivebox version
-
-      - name: Test archivebox init
-        run: |
-          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
-          archivebox init
-
-      - name: Test archivebox status
-        run: |
-          cd /tmp/archivebox-test
-          archivebox status
-
-  # On release only: generate the real formula with PyPI URL and push to tap
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: build-and-test
-    runs-on: macos-latest
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Wait for PyPI package availability
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..."
-          for i in $(seq 1 30); do
-            if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then
-              echo "[√] archivebox==${VERSION} is available on PyPI"
-              break
-            fi
-            if [ "$i" -eq 30 ]; then
-              echo "[!] Timed out waiting for PyPI. Trying to install anyway..."
-              break
-            fi
-            echo "    Attempt $i/30 - not yet available, waiting 30s..."
-            sleep 30
-          done
-
-      - name: Generate release formula via build_brew.sh
-        run: ./bin/build_brew.sh
-
-      - name: Test formula install
-        run: |
-          brew install --build-from-source brew_dist/archivebox.rb
-          archivebox version
-
-      - name: Upload formula artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox.rb
-          path: brew_dist/archivebox.rb
-
-      - name: Push to homebrew-archivebox tap
-        env:
-          GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap
-
-          cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \
-            cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb
-
-          cd /tmp/tap
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add -A
-          if git diff --cached --quiet; then
-            echo "[i] No changes to formula, skipping push."
-          else
-            git commit -m "Update archivebox to v${VERSION}"
-            git push origin HEAD
-            echo "[√] Formula pushed to homebrew-archivebox tap"
-          fi

.github/workflows/lint.yml

@@ -22,13 +22,13 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -39,12 +39,13 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - name: Install dependencies with uv
         run: |
-          uv sync --extra sonic --extra debug --all-groups --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --all-groups -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[sonic,debug]"
 
       - name: Run prek
         run: uv run --no-sync --no-sources prek run --all-files

.github/workflows/pip.yml

@@ -22,12 +22,14 @@ jobs:
       id-token: write
 
     runs-on: ubuntu-24.04
+    environment: pypi
     steps:
       - uses: actions/checkout@v4
 
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - name: Set up Python
@@ -42,8 +44,89 @@ jobs:
           packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
           version: 1.0
 
+      - name: Wait for released ArchiveBox deps on PyPI
+        run: |
+          python - <<'PY'
+          import re
+          import sys
+          import time
+          import tomllib
+          import urllib.request
+
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  try:
+                      with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20):
+                          available = True
+                  except Exception:
+                      available = False
+                  if available:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
+          PY
+
+      - name: Prepare released dependency sync project
+        run: |
+          python - <<'PY'
+          from pathlib import Path
+          import json
+          import re
+          import urllib.request
+
+          source = Path("pyproject.toml")
+          target_dir = Path(".ci-uv-project")
+          target_dir.mkdir(exist_ok=True)
+          text = source.read_text()
+          text = text.replace(
+              'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
+              'environments = ["sys_platform == \'linux\'"]',
+          )
+
+          # CI runs immediately after publishing internal abx packages. PyPI's
+          # simple index can lag the version JSON endpoint, so sync against the
+          # exact wheel URLs from the version endpoint while leaving package
+          # metadata in pyproject.toml unchanged for the actual build.
+          for package in ("abxpkg", "abx-plugins", "abx-dl"):
+              match = re.search(rf'"{re.escape(package)}>=(?P<version>[^"]+)"', text)
+              if not match:
+                  continue
+              version = match.group("version")
+              with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
+                  data = json.load(response)
+              wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
+              text = re.sub(
+                  rf'"{re.escape(package)}>=[^"]+"',
+                  f'"{package} @ {wheel_url}"',
+                  text,
+                  count=1,
+              )
+
+          (target_dir / "pyproject.toml").write_text(text)
+          PY
+
       - name: UV install archivebox dev + run sub-dependencies
-        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+        env:
+          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
+        run: uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
 
       - name: UV build archivebox and archivebox/pkgs/* packages
         run: |
@@ -53,17 +136,22 @@ jobs:
         uses: pypa/gh-action-pypi-publish@release/v1
         # ignore when publish to PyPI fails due to duplicate tag
         continue-on-error: true
-        with:
-          password: ${{ secrets.PYPI_PAT_SECRET }}
 
       - name: UV install archivebox and archivebox/pkgs/* locally for tests
-        run: uv sync --all-extras --no-sources --no-cache
-
-      - name: UV run archivebox init + archivebox version
+        env:
+          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
         run: |
-          mkdir -p data && cd data
-          uv run --no-sync --no-sources archivebox init \
-          && uv run --no-sync --no-sources archivebox version
-          # && uv run archivebox add 'https://example.com' \
-          # && uv run archivebox status \
-          # || (echo "UV Failed to run archivebox!" && exit 1)
+          uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+          uv pip install --python .venv/bin/python --no-deps dist/archivebox-*.whl
+
+      - name: Verify built package full install
+        env:
+          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
+        run: |
+          set -Eeuo pipefail
+          DATA_DIR="$(mktemp -d)"
+          cd "$DATA_DIR"
+          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox init
+          timeout 30m uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox install
+          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox version
+          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox status

.github/workflows/release-runner.yml

@@ -3,26 +3,30 @@ name: Release State
 on:
   push:
     branches:
-      - '**'
+      - dev
   workflow_dispatch:
 
 permissions:
+  actions: write
   contents: write
   id-token: write
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  cancel-in-progress: false
 
 jobs:
   release-state:
+    if: github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'release:')
     runs-on: ubuntu-latest
+    environment: pypi
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: true
           ref: ${{ github.ref_name }}
+          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
 
       - uses: actions/setup-python@v5
         with:
@@ -30,6 +34,7 @@ jobs:
 
       - uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - uses: actions/setup-node@v4
@@ -43,7 +48,13 @@ jobs:
 
       - name: Run release script
         env:
-          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-          GH_TOKEN: ${{ github.token }}
-          PYPI_PAT_SECRET: ${{ secrets.PYPI_PAT_SECRET }}
+          DEFAULT_BRANCH: dev
+          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
         run: ./bin/release.sh
+
+      - name: Dispatch image and tap updates
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
+        run: |
+          gh workflow run docker.yml --ref dev
+          gh workflow run update-homebrew-tap.yml --ref dev

.github/workflows/release.yml

@@ -2,8 +2,8 @@ name: Release
 
 # Orchestrates the full release pipeline:
 #   1. Build and publish pip package to PyPI
-#   2. Build .deb packages and Homebrew formula (in parallel, after pip)
-#   3. Build Docker images (in parallel with deb/brew)
+#   2. Build Homebrew formula after pip
+#   3. Build Docker images in parallel
 #
 # Individual workflows also run on push for CI (see their own triggers).
 # Manual fallback for the hosted release pipeline. The normal prerelease loop
@@ -28,12 +28,6 @@ jobs:
     uses: ./.github/workflows/pip.yml
     secrets: inherit
 
-  debian:
-    name: Build .deb packages
-    needs: pip
-    uses: ./.github/workflows/debian.yml
-    secrets: inherit
-
   homebrew:
     name: Update Homebrew formula
     needs: pip

.github/workflows/test-parallel.yml

@@ -16,6 +16,7 @@ env:
   PYTHONLEGACYWINDOWSSTDIO: utf-8
   USE_COLOR: False
   UV_NO_SOURCES: "1"
+  ABXPKG_MIN_RELEASE_AGE: "0"
 
 jobs:
   discover-tests:
@@ -49,8 +50,16 @@ jobs:
 
             # Extract a display name for the test
             name="main/$(basename $test_file .py | sed 's/^test_//')"
+            needs_chromium=false
+            if grep -Eiq '(chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue)' "$test_file"; then
+              needs_chromium=true
+            fi
+            needs_sonic=false
+            if grep -Eiq "shutil\\.which\\([\"']sonic|SEARCH_BACKEND_ENGINE=.*sonic|worker_sonic" "$test_file"; then
+              needs_sonic=true
+            fi
 
-            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
           done
           json_array+="]"
 
@@ -59,7 +68,7 @@ jobs:
           echo "$json_array" | jq '.'
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Discover plugin tests
         id: set-plugin-matrix
@@ -79,8 +88,12 @@ jobs:
             if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
               needs_chromium=true
             fi
+            needs_sonic=false
+            if [ "$plugin_name" = "search_backend_sonic" ]; then
+              needs_sonic=true
+            fi
 
-            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium}"
+            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
           done
           json_array+="]"
 
@@ -112,13 +125,13 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       - name: Set up Python ${{ matrix.python }}
         uses: actions/setup-python@v5
@@ -129,7 +142,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -151,13 +164,85 @@ jobs:
 
       - name: Install dependencies with uv
         run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
+
+      - name: Install Chrome through ArchiveBox
+        if: ${{ matrix.test.needs_chromium }}
+        env:
+          MIN_CHROMIUM_MAJOR: "149"
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
+          case "$major" in
+            ''|*[!0-9]*) major=0 ;;
+          esac
+          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
+            exit 1
+          fi
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
+          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "DATA_DIR=$ABX_CI_DATA_DIR" >> "$GITHUB_ENV"
+          echo "NODE_MODULES_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules" >> "$GITHUB_ENV"
+          echo "NODE_PATH=$LIB_DIR/pnpm/packages/chrome/node_modules" >> "$GITHUB_ENV"
+          echo "PNPM_BIN_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules/.bin" >> "$GITHUB_ENV"
+          echo "NPM_BIN_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules/.bin" >> "$GITHUB_ENV"
+          echo "CHROMEWEBSTORE_EXTENSIONS_DIR=$LIB_DIR/chromewebstore/extensions" >> "$GITHUB_ENV"
+
+      - name: Install Sonic through ArchiveBox
+        if: ${{ matrix.test.needs_sonic }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
+          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
 
       - name: Run test - ${{ matrix.test.name }}
         run: |
           mkdir -p tests/out
+          set +e
           uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
+          status=$?
+          set -e
+          if [ "$status" -eq 5 ]; then
+            echo "No tests collected from ${{ matrix.test.path }}; treating as an empty test module."
+            exit 0
+          fi
+          exit "$status"
 
   plugin-tests:
     name: ${{ matrix.plugin.name }}
@@ -182,13 +267,13 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       - name: Set up Python 3.13
         uses: actions/setup-python@v5
@@ -199,7 +284,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -221,58 +306,28 @@ jobs:
 
       - name: Install dependencies with uv
         run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
 
-      - name: Select Chromium for extension-capable CDP
+      - name: Install Chrome through ArchiveBox
         if: ${{ matrix.plugin.needs_chromium }}
         env:
           MIN_CHROMIUM_MAJOR: "149"
           LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
         run: |
           set -euo pipefail
-          mkdir -p "$LIB_DIR"
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
           echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
           echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
 
-          candidate="/usr/bin/chromium"
-          version=""
-          major=0
-          if [ -x "$candidate" ]; then
-            version="$("$candidate" --version || true)"
-            major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
-            case "$major" in
-              ''|*[!0-9]*) major=0 ;;
-            esac
-          fi
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
 
-          if [ "$major" -ge "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Using runner Chromium: $candidate ($version)"
-            echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
-            exit 0
-          fi
-
-          echo "Runner Chromium is unavailable or too old: ${version:-not found}"
-          echo "Installing Playwright Chromium into $LIB_DIR..."
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=npm \
-            --lib="$LIB_DIR" \
-            --postinstall-scripts=false \
-            --min-release-age=0 \
-            --no-cache \
-            --overrides='{"npm":{"install_args":["playwright@next"]}}' \
-            playwright
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=playwright \
-            --lib="$LIB_DIR" \
-            --bin-dir="$LIB_DIR/env/bin" \
-            --min-version="$MIN_CHROMIUM_MAJOR" \
-            chromium
-
-          candidate="$LIB_DIR/env/bin/chromium"
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
           if [ ! -x "$candidate" ]; then
-            echo "Playwright Chromium shim was not created at $candidate" >&2
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
             exit 1
           fi
           version="$("$candidate" --version || true)"
@@ -281,11 +336,42 @@ jobs:
             ''|*[!0-9]*) major=0 ;;
           esac
           if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Playwright Chromium is still too old: ${version:-unknown}" >&2
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
             exit 1
           fi
-          echo "Using Playwright Chromium: $candidate ($version)"
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
           echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "DATA_DIR=$ABX_CI_DATA_DIR" >> "$GITHUB_ENV"
+          echo "NODE_MODULES_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules" >> "$GITHUB_ENV"
+          echo "NODE_PATH=$LIB_DIR/pnpm/packages/chrome/node_modules" >> "$GITHUB_ENV"
+          echo "PNPM_BIN_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules/.bin" >> "$GITHUB_ENV"
+          echo "NPM_BIN_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules/.bin" >> "$GITHUB_ENV"
+          echo "CHROMEWEBSTORE_EXTENSIONS_DIR=$LIB_DIR/chromewebstore/extensions" >> "$GITHUB_ENV"
+
+      - name: Install Sonic through ArchiveBox
+        if: ${{ matrix.plugin.needs_sonic }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
+          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
 
       - name: Run plugin tests - ${{ matrix.plugin.name }}
         env:

.github/workflows/test.yml

@@ -31,13 +31,13 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       ### Setup Python & JS Languages
       - name: Set up Python ${{ matrix.python }}
@@ -49,7 +49,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v4
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -72,8 +72,8 @@ jobs:
 
       - name: Install dependencies with uv
         run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
 
       ### Run the tests
       - name: Directory listing for debugging
@@ -81,10 +81,15 @@ jobs:
           pwd
           ls
 
-      - name: Archivebox version
+      - name: ArchiveBox full install check
         run: |
-          mkdir -p tests/out/data
-          DATA_DIR="$PWD/tests/out/data" uv run --no-sync --no-sources archivebox version
+          set -Eeuo pipefail
+          DATA_DIR="$PWD/tests/out/data"
+          mkdir -p "$DATA_DIR"
+          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox init
+          timeout 30m uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox install
+          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox version
+          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox status
 
       - name: Test built package with pytest
         # TODO: remove this exception for windows once we get tests passing on that platform

.github/workflows/update-homebrew-tap.yml

@@ -0,0 +1,37 @@
+name: Update downstream packages
+
+on:
+  push:
+    branches: [dev]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: update-downstream-packages-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dispatch:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Dispatch downstream package updates
+        env:
+          DOWNSTREAM_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || secrets.HOMEBREW_TAP_TOKEN }}
+        run: |
+          if [ -z "$DOWNSTREAM_TOKEN" ]; then
+            echo "RELEASE_GH_TOKEN/HOMEBREW_TAP_TOKEN is not configured; downstream schedules will update packages."
+            exit 0
+          fi
+
+          for repo in homebrew-archivebox debian-archivebox; do
+            curl --fail-with-body \
+              -X POST \
+              -H "Accept: application/vnd.github+json" \
+              -H "Authorization: Bearer ${DOWNSTREAM_TOKEN}" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              "https://api.github.com/repos/ArchiveBox/${repo}/dispatches" \
+              -d '{"event_type":"archivebox-dev-updated","client_payload":{"ref":"dev"}}'
+          done

AGENTS.md

@@ -0,0 +1,87 @@
+# ArchiveBox Agent Guide
+
+ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev` branch.
+
+## Shared Standards
+
+- Use `uv` and `uv run` for Python commands. Do not use system `python`, direct `.venv/bin/python`, or `pip` commands.
+- Prefer existing repo patterns, helper APIs, fixtures, scripts, and command surfaces.
+- Keep edits focused and minimal. Do not add wrappers, shims, aliases, or extra abstraction layers unless the current code path requires them.
+- Do not weaken assertions, skip tests, xfail tests, or accept flaky behavior.
+- No mocks, monkeypatches, fakes, simulated handlers, fake binaries, fake hooks, fake buses, or direct shortcuts around user-facing flows.
+- Tests and verification should use real CLI commands, REST/API calls, browser UI flows, real hooks, real installs, real subprocesses, real DB rows, real files, and existing fixtures.
+- Assertions must verify real correctness: exit codes, returned values, DB state, filesystem contents, field values, rendered output, and side effects.
+- Start behavior fixes with a red failing test when a test is requested or practical.
+- Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
+- Read `README.md` for the full setup, CLI, Docker, API, and release surface.
+
+## Development Setup
+
+```bash
+uv sync --dev --all-extras
+mkdir -p data
+cd data
+uv run --project .. archivebox init --install
+```
+
+Run collection commands from inside an initialized data directory:
+
+```bash
+cd data
+uv run --project .. archivebox status
+uv run --project .. archivebox add 'https://example.com'
+uv run --project .. archivebox run
+```
+
+## User-Facing Setup
+
+Recommended CLI install:
+
+```bash
+uv tool install archivebox
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init --install
+archivebox add 'https://example.com'
+archivebox server 0.0.0.0:8000
+```
+
+Alternative install methods:
+
+- Docker Compose / Docker
+- Homebrew
+- Debian package
+- pip
+
+## Basic Usage
+
+```bash
+archivebox version
+archivebox help
+archivebox status
+archivebox install
+archivebox add 'https://example.com'
+archivebox add --extract=title,screenshot,pdf 'https://example.com'
+archivebox list --json --with-headers
+archivebox search 'example'
+archivebox update --filter-type=domain example.com
+archivebox remove --filter-type=exact 'https://example.com'
+archivebox run
+```
+
+## Verification
+
+Use targeted tests for focused work:
+
+<!--pytest.mark.skip(reason="pytest invocation")-->
+```bash
+uv run pytest archivebox/tests/test_cli_add.py -q
+uv run prek run --all-files
+```
+
+Use the full release/deploy loop only when requested:
+
+<!--pytest.mark.skip(reason="release/deploy script")-->
+```bash
+./bin/release_dev_stack.sh
+```

CLAUDE.md

@@ -1,497 +0,0 @@
-# Claude Code Development Guide for ArchiveBox
-
-## Quick Start
-
-```bash
-# Set up dev environment (always use uv, never pip directly)
-uv sync --dev --all-extras
-
-# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
-```
-
-## Development Environment Setup
-
-### Prerequisites
-- Python 3.11+ (3.13 recommended)
-- uv package manager
-- A non-root user for running tests (e.g., `testuser`)
-
-### Install Dependencies
-```bash
-uv sync --dev --all-extras  # Always use uv, never pip directly
-```
-
-### Activate Virtual Environment
-```bash
-source .venv/bin/activate
-```
-
-### Common Gotchas
-
-#### File Permissions
-New files created by root need permissions fixed for testuser:
-```bash
-chmod 644 archivebox/tests/test_*.py
-```
-
-#### DATA_DIR Environment Variable
-ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
-
-## Code Style Guidelines
-
-### Naming Conventions for Grep-ability
-Use consistent naming for everything to enable easy grep-ability and logical grouping:
-
-**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
-
-**Examples**:
-```python
-# Filesystem migration methods - all start with fs_
-def fs_migration_needed() -> bool: ...
-def fs_migrate() -> None: ...
-def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
-def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
-def _fs_next_version(current: str) -> str: ...
-
-# Logging methods - ALL must start with log_ or _log
-def log_migration_start(snapshot_id: str) -> None: ...
-def _log_error(message: str) -> None: ...
-def log_validation_result(ok: bool, msg: str) -> None: ...
-```
-
-**Rules**:
-- Group related functions with common prefixes
-- Use `_` prefix for internal/private helpers within the same family
-- ALL logging-related methods MUST start with `log_` or `_log`
-- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
-- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
-
-### Minimize Unique Names and Data Structures
-**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
-
-**Example - GOOD**:
-```python
-# Binary has overrides field
-binary = Binary(overrides={'TIMEOUT': '60s'})
-
-# Binary reuses the same field name and structure
-class Binary(models.Model):
-    overrides = models.JSONField(default=dict)  # Same name, same structure
-```
-
-**Example - BAD**:
-```python
-# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
-class Binary(models.Model):
-    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
-```
-
-**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
-
-## Testing
-
-### CRITICAL: Never Run as Root
-ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
-
-```bash
-# Run all migration tests
-sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
-
-# Run specific test file
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
-
-# Run single test
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
-```
-
-### Test File Structure
-```
-archivebox/tests/
-├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
-├── test_migrations_fresh.py      # Fresh install tests
-├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
-├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
-└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
-```
-
-### Test Writing Standards
-
-#### NO MOCKS - Real Tests Only
-Tests must exercise real code paths:
-- Create real SQLite databases with version-specific schemas
-- Seed with realistic test data
-- Run actual `python -m archivebox` commands via subprocess
-- Query SQLite directly to verify results
-
-**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
-
-#### NO SKIPS
-Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
-
-#### Strict Assertions
-- `init` command must return exit code 0 (not `[0, 1]`)
-- Verify ALL data is preserved, not just "at least one"
-- Use exact counts (`==`) not loose bounds (`>=`)
-
-### Example Test Pattern
-```python
-def test_migration_preserves_snapshots(self):
-    """Migration should preserve all snapshots."""
-    result = run_archivebox(self.work_dir, ['init'], timeout=45)
-    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
-
-    ok, msg = verify_snapshot_count(self.db_path, expected_count)
-    self.assertTrue(ok, msg)
-```
-
-### Testing Gotchas
-
-#### Extractors Disabled for Speed
-Tests disable all extractors via environment variables for faster execution:
-```python
-env['SAVE_TITLE'] = 'False'
-env['SAVE_FAVICON'] = 'False'
-# ... etc
-```
-
-#### Timeout Settings
-Use appropriate timeouts for migration tests (45s for init, 60s default).
-
-### Plugin Testing & Code Coverage
-
-**Target: 80-90% coverage** for critical plugins (screenshot, chrome, singlefile, dom)
-
-```bash
-# Run plugin tests with coverage (both Python + JavaScript)
-bash bin/test_plugins.sh screenshot
-
-# View coverage reports
-bash bin/test_plugins.sh --coverage-report
-# Or individual reports:
-coverage report --show-missing --include='archivebox/plugins/*' --omit='*/tests/*'
-```
-
-#### Plugin Test Structure
-
-Tests are **completely isolated** from ArchiveBox - they replicate production directory structure in temp dirs:
-
-```python
-# Correct production paths:
-# Crawl:    DATA_DIR/users/{username}/crawls/YYYYMMDD/example.com/{crawl-id}/{plugin}/
-# Snapshot: DATA_DIR/users/{username}/snapshots/YYYYMMDD/example.com/{snapshot-uuid}/{plugin}/
-
-with tempfile.TemporaryDirectory() as tmpdir:
-    data_dir = Path(tmpdir)
-
-    # Crawl-level plugin (e.g., chrome launcher)
-    crawl_dir = data_dir / 'users' / 'testuser' / 'crawls' / '20240101' / 'example.com' / 'crawl-123'
-    chrome_dir = crawl_dir / 'chrome'
-    chrome_dir.mkdir(parents=True)
-
-    # Snapshot-level plugin (e.g., screenshot)
-    snapshot_dir = data_dir / 'users' / 'testuser' / 'snapshots' / '20240101' / 'example.com' / 'snap-456'
-    screenshot_dir = snapshot_dir / 'screenshot'
-    screenshot_dir.mkdir(parents=True)
-
-    # Run hook in its output directory
-    result = subprocess.run(
-        ['node', str(SCREENSHOT_HOOK), '--url=https://example.com'],
-        cwd=str(screenshot_dir),
-        env={**get_test_env(), 'EXTRA_CONTEXT': '{"snapshot_id":"snap-456"}'},
-        capture_output=True,
-        timeout=120
-    )
-```
-
-#### Coverage Improvement Loop
-
-To improve from ~20% to 80%+:
-
-1. **Run tests**: `bash bin/test_plugins.sh screenshot` → Shows: `19.1% (13/68 ranges)`
-2. **Identify gaps**: Check hook file for untested paths (session connection vs fallback, config branches, error cases)
-3. **Add tests**: Test both execution paths (connect to session + launch own browser), skip conditions, error cases, config variations
-4. **Verify**: Re-run tests → Should show: `85%+ (58+/68 ranges)`
-
-**Critical**: JavaScript hooks have TWO paths that both must be tested (connect to session ~50% + launch browser ~30% + shared ~20%). Testing only one path = max 50% coverage possible!
-
-## Database Migrations
-
-### Generate and Apply Migrations
-```bash
-# Generate migrations (run from archivebox subdirectory)
-cd archivebox
-./manage.py makemigrations
-
-# Apply migrations to test database
-cd data/
-archivebox init
-```
-
-### Schema Versions
-- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
-- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
-- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
-- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
-
-### Testing a Migration Path
-1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
-2. Seed with realistic test data using `seed_0_X_data()`
-3. Run `archivebox init` to trigger migrations
-4. Verify data preservation with `verify_*` functions
-5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
-
-### Squashed Migrations
-When testing 0.8.x (dev branch), you must record ALL replaced migrations:
-```python
-# The squashed migration replaces these - all must be recorded
-('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
-('core', '0024_auto_20240513_1143'),
-# ... all 52 migrations from 0023-0074 ...
-('core', '0023_new_schema'),  # Also record the squashed migration itself
-```
-
-### Migration Strategy
-- Squashed migrations for clean installs
-- Individual migrations recorded for upgrades from dev branch
-- `replaces` attribute in squashed migrations lists what they replace
-
-### Migration Gotchas
-
-#### Circular FK References in Schemas
-SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
-
-## Plugin System Architecture
-
-### Plugin Dependency Rules
-
-Like other plugins, chrome plugins **ARE NOT ALLOWED TO DEPEND ON ARCHIVEBOX OR DJANGO**.
-However, they are allowed to depend on two shared files ONLY:
-- `archivebox/plugins/chrome/chrome_utils.js` ← source of truth API for all basic chrome ops
-- `archivebox/plugins/chrome/tests/chrome_test_utils.py` ← use for your tests, do not implement launching/killing/pid files/cdp/etc. in python, just extend this file as needed.
-
-### Chrome-Dependent Plugins
-
-Many plugins depend on Chrome/Chromium via CDP (Chrome DevTools Protocol). When checking for script name references or debugging Chrome-related issues, check these plugins:
-
-**Main puppeteer-based chrome installer + launcher plugin**:
-- `chrome` - Core Chrome integration (CDP, launch, navigation)
-
-**Metadata extraction using chrome/chrome_utils.js / CDP**:
-- `dns` - DNS resolution info
-- `ssl` - SSL certificate info
-- `headers` - HTTP response headers
-- `redirects` - Capture redirect chains
-- `staticfile` - Direct file downloads (e.g. if the url itself is a .png, .exe, .zip, etc.)
-- `responses` - Capture network responses
-- `consolelog` - Capture console.log output
-- `title` - Extract page title
-- `accessibility` - Extract accessibility tree
-- `seo` - Extract SEO metadata
-
-**Extensions installed using chrome/chrome_utils.js / controlled using CDP**:
-- `ublock` - uBlock Origin ad blocking
-- `istilldontcareaboutcookies` - Cookie banner dismissal
-- `twocaptcha` - 2captcha CAPTCHA solver integration
-
-**Page-alteration plugins to prepare the content for archiving**:
-- `modalcloser` - Modal dialog dismissal
-- `infiniscroll` - Infinite scroll handler
-
-**Main Extractor Outputs**:
-- `dom` - DOM snapshot extraction
-- `pdf` - Generate PDF snapshots
-- `screenshot` - Generate screenshots
-- `singlefile` - SingleFile archival, can be single-file-cli that launches chrome, or singlefile extension running inside chrome
-
-**Crawl URL parsers** (post-process dom.html, singlefile.html, staticfile, responses, headers, etc. for URLs to re-emit as new queued Snapshots during recursive crawling):
-- `parse_dom_outlinks` - Extract outlinks from DOM (special, uses CDP to directly query browser)
-- `parse_html_urls` - Parse URLs from HTML (doesn't use chrome directly, just reads dom.html)
-- `parse_jsonl_urls` - Parse URLs from JSONL (doesn't use chrome directly, just reads dom.html)
-- `parse_netscape_urls` - Parse Netscape bookmark format (doesn't use chrome directly, just reads dom.html)
-
-### Finding Chrome-Dependent Plugins
-
-```bash
-# Find all files containing "chrom" (case-insensitive)
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d: -f1 | sort -u
-
-# Or get just the plugin names
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d/ -f3 | sort -u
-```
-
-**Note**: This list may not be complete. Always run the grep command above when checking for Chrome-related script references or debugging Chrome integration issues.
-
-## Architecture Notes
-
-### Crawl Model (0.9.x)
-- Crawl groups multiple Snapshots from a single `add` command
-- Each `add` creates one Crawl with one or more Snapshots
-- Seed model was removed - crawls now store URLs directly
-
-## Code Coverage
-
-### Overview
-
-Coverage tracking is enabled for passive collection across all contexts:
-- Unit tests (pytest)
-- Integration tests
-- Dev server (manual testing)
-- CLI usage
-
-Coverage data accumulates in `.coverage` file and can be viewed/analyzed to find dead code.
-
-### Install Coverage Tools
-
-```bash
-uv sync --dev  # Installs pytest-cov and coverage
-```
-
-### Running with Coverage
-
-#### Unit Tests
-```bash
-# Run tests with coverage
-pytest --cov=archivebox --cov-report=term archivebox/tests/
-
-# Or run specific test file
-pytest --cov=archivebox --cov-report=term archivebox/tests/test_migrations_08_to_09.py
-```
-
-#### Dev Server with Coverage
-```bash
-# Start dev server with coverage tracking
-coverage run --parallel-mode -m archivebox server
-
-# Or CLI commands
-coverage run --parallel-mode -m archivebox init
-coverage run --parallel-mode -m archivebox add https://example.com
-```
-
-#### Manual Testing (Always-On)
-To enable coverage during ALL Python executions (passive tracking):
-
-```bash
-# Option 1: Use coverage run wrapper
-coverage run --parallel-mode -m archivebox [command]
-
-# Option 2: Set environment variable (tracks everything)
-export COVERAGE_PROCESS_START=pyproject.toml
-# Now all Python processes will track coverage
-archivebox server
-archivebox add https://example.com
-```
-
-### Viewing Coverage
-
-#### Text Report (Quick View)
-```bash
-# Combine all parallel coverage data
-coverage combine
-
-# View summary
-coverage report
-
-# View detailed report with missing lines
-coverage report --show-missing
-
-# View specific file
-coverage report --include="archivebox/core/models.py" --show-missing
-```
-
-#### JSON Report (LLM-Friendly)
-```bash
-# Generate JSON report
-coverage json
-
-# View the JSON
-cat coverage.json | jq '.files | keys'  # List all files
-
-# Find files with low coverage
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 50) | "\(.key): \(.value.summary.percent_covered)%"'
-
-# Find completely uncovered files (dead code candidates)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# Get missing lines for a specific file
-cat coverage.json | jq '.files["archivebox/core/models.py"].missing_lines'
-```
-
-#### HTML Report (Visual)
-```bash
-# Generate interactive HTML report
-coverage html
-
-# Open in browser
-open htmlcov/index.html
-```
-
-### Isolated Runs
-
-To measure coverage for specific scenarios:
-
-```bash
-# 1. Reset coverage data
-coverage erase
-
-# 2. Run your isolated test/scenario
-pytest --cov=archivebox archivebox/tests/test_migrations_fresh.py
-# OR
-coverage run --parallel-mode -m archivebox add https://example.com
-
-# 3. View results
-coverage combine
-coverage report --show-missing
-
-# 4. Optionally export for analysis
-coverage json
-```
-
-### Finding Dead Code
-
-```bash
-# 1. Run comprehensive tests + manual testing to build coverage
-pytest --cov=archivebox archivebox/tests/
-coverage run --parallel-mode -m archivebox server  # Use the app manually
-coverage combine
-
-# 2. Find files with 0% coverage (strong dead code candidates)
-coverage json
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# 3. Find files with <10% coverage (likely dead code)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 10) | "\(.key): \(.value.summary.percent_covered)%"' | sort -t: -k2 -n
-
-# 4. Generate detailed report for analysis
-coverage report --show-missing > coverage_report.txt
-```
-
-### Tips
-
-- **Parallel mode** (`--parallel-mode`): Allows multiple processes to track coverage simultaneously without conflicts
-- **Combine**: Always run `coverage combine` before viewing reports to merge parallel data
-- **Reset**: Use `coverage erase` to start fresh for isolated measurements
-- **Branch coverage**: Enabled by default - tracks if both branches of if/else are executed
-- **Exclude patterns**: Config in `pyproject.toml` excludes tests, migrations, type stubs
-
-## Debugging Tips
-
-### Check Migration State
-```bash
-sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
-```
-
-### Check Table Schema
-```bash
-sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
-```
-
-### Verbose Test Output
-```bash
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
-```
-
-### Kill Zombie Chrome Processes
-```bash
-./bin/kill_chrome.sh
-```

Dockerfile

@@ -1,38 +1,188 @@
-# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
-#     python3.13, uv, python3-ldap
-#     curl, wget, git, dig, ping, tree, nano
-#     node, npm
-#     ArchiveBox and plugin runtime dependencies installed by archivebox init --install
-# Usage:
-#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
-#     docker build . -t archivebox
-#     docker run -v "$PWD/data":/data archivebox init
-#     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
-#     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
-#     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
-#     docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
-# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+# syntax=docker/dockerfile:1.7
 
+# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
+# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
+# ArchiveBox owns sonic, supervisor, Django, and the app runtime.
+# Build abx-dl first, then point this file at it:
+#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
+#       --build-context abxbus=../abxbus \
+#       --build-context abxpkg=../abxpkg \
+#       --build-context abx-plugins=../abx-plugins \
+#       -t archivebox/abx-dl:dev
+#   docker buildx build . -f Dockerfile \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
+#       -t archivebox:multistage
 
-#########################################################################################
+ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
 
-### Example: Using ArchiveBox in your own project's Dockerfile ########
-
-# FROM python:3.13-slim
-# WORKDIR /data
-# RUN pip install archivebox>=0.9.0   # use latest release here
-# RUN archivebox install
-# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
-
-#########################################################################################
+FROM archivebox/sonic:1.4.9 AS sonic
+FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
 
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
-ARG TARGETVARIANT=
+ARG TARGETVARIANT
+ARG ARCHIVEBOX_COMMIT_HASH=""
 
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ubuntu:24.04
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_COMPILE=1 \
+    PIP_ONLY_BINARY=aiohttp \
+    npm_config_loglevel=error
+
+ENV PYTHON_VERSION=3.13 \
+    NODE_VERSION=24
+
+ENV ARCHIVEBOX_USER=archivebox \
+    DEFAULT_ARCHIVEBOX_UID=911 \
+    DEFAULT_ARCHIVEBOX_GID=911 \
+    IN_DOCKER=True
+
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    CONFIG_DIR=/opt/archivebox \
+    ABXPKG_LIB_DIR=/opt/archivebox/lib \
+    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
+    PERSONAS_DIR=/data/personas \
+    CHROME_HEADLESS=true \
+    CHROME_SANDBOX=false \
+    CHROME_ISOLATION=crawl \
+    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
+
+ENV TMP_DIR=/tmp/archivebox \
+    PIP_VENV_PYTHON=/venv/bin/python3 \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no
+
+ENV HOME=/home/archivebox \
+    XDG_CONFIG_HOME=/home/archivebox/.config \
+    XDG_CACHE_HOME=/opt/archivebox/lib/cache \
+    ABXPKG_INSTALL_TIMEOUT=600 \
+    ABXPKG_POSTINSTALL_SCRIPTS=True \
+    ABXPKG_MIN_RELEASE_AGE=0 \
+    TIMEOUT=600
+
+ENV UV_COMPILE_BYTECODE=false \
+    UV_PYTHON_PREFERENCE=managed \
+    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv \
+    VIRTUAL_ENV=/venv \
+    PATH="/venv/bin:/opt/node/bin:$PATH"
+
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
+WORKDIR "$CODE_DIR"
+
+RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
+    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && uname -a \
+    && sed -n '1,7p' /etc/os-release \
+    && abxpkg load --binproviders=env node \
+    && abxpkg load --binproviders=env uv \
+    ) | tee -a /VERSION.txt
+
+FROM archivebox-runtime-base AS archivebox-builder
+
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    <<'EOF'
+echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
+echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache
+echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends
+echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests
+rm -f /etc/apt/apt.conf.d/docker-clean
+apt-get update -qq
+apt-get install -qq -y --no-install-recommends \
+    build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+/usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}"
+/usr/bin/uv pip install setuptools pip wheel
+
+mkdir -p /tmp/archivebox-uv-project
+/venv/bin/python - <<'PY'
+from pathlib import Path
+import json
+import re
+import urllib.request
+
+source = Path("/app/pyproject.toml")
+target = Path("/tmp/archivebox-uv-project/pyproject.toml")
+text = source.read_text()
+text = text.replace(
+    'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
+    'environments = ["sys_platform == \'linux\'"]',
+)
+
+# Docker builds need the just-published internal abx wheels immediately, but
+# PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a
+# Docker-only dependency view from the version JSON so the published package
+# metadata stays normal while image builds remain resumable after a release.
+for package in ("abxpkg", "abx-plugins", "abx-dl"):
+    match = re.search(rf'"{re.escape(package)}>=(?P<version>[^"]+)"', text)
+    if not match:
+        continue
+    version = match.group("version")
+    with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
+        data = json.load(response)
+    wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
+    text = re.sub(
+        rf'"{re.escape(package)}>=[^"]+"',
+        f'"{package} @ {wheel_url}"',
+        text,
+        count=1,
+    )
+
+target.write_text(text)
+PY
+
+/usr/bin/uv sync \
+    --project /tmp/archivebox-uv-project \
+    --refresh \
+    --no-dev \
+    --inexact \
+    --no-install-project \
+    --no-install-workspace \
+    --no-sources
+(find /venv/lib/python3.*/site-packages -type f -name '*.so' -exec strip --strip-unneeded {} + 2>/dev/null || true)
+rm -f /venv/bin/uv /venv/bin/uvx
+apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+apt-get autoremove -y
+rm -rf /var/lib/apt/lists/*
+EOF
+
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && COMMIT_HASH="$( \
+        if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+            echo "$ARCHIVEBOX_COMMIT_HASH"; \
+        elif [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
+            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
+            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+                echo "$HEAD_REF"; \
+            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
+                REF_PATH="${HEAD_REF#ref: }"; \
+                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
+            fi; \
+        fi)" \
+    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
+    && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
+    && rm -f /venv/bin/uv /venv/bin/uvx \
+    && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
+
+FROM archivebox-runtime-base
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -43,346 +193,70 @@ LABEL name="archivebox" \
     org.opencontainers.image.vendor="ArchiveBox" \
     org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
     org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile" \
-    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
-    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
-    com.docker.desktop.extension.api.version=">= 1.4.7" \
-    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
-    com.docker.extension.publisher-url="https://archivebox.io" \
-    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
-    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
-    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
-    com.docker.extension.categories='database,utility-tools'
+    com.docker.image.source.entrypoint="Dockerfile"
 
-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-######### Environment Variables #################################
-
-# Global build-time and runtime environment constants + default pkg manager config
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    npm_config_loglevel=error
-
-# Language Version config
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22.22.3
-
-# Non-root User config
-ENV ARCHIVEBOX_USER="archivebox" \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
-
-# ArchiveBox Source Code + Lib + Data paths
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
-    LIB_BIN_DIR=/opt/archivebox/lib/bin \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/browsers
-
-# Bash SHELL config
-# http://redsymbol.net/articles/unofficial-bash-strict-mode/
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
-
-######### System Environment ####################################
-
-# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
-WORKDIR "$CODE_DIR"
-
-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-# Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && echo \
-    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
-    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
-    && echo \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && which bash && bash --version | sed -n '1p' \
-    && which dpkg && dpkg --version | sed -n '1p' \
-    && echo -e '\n\n' && env && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-# Create non-privileged user for archivebox and chrome
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system $ARCHIVEBOX_USER \
-    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
-    | tee -a /VERSION.txt
-    # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
-    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
-
-# Install system apt dependencies (adding backports to access more recent apt updates)
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
-    && mkdir -p /etc/apt/keyrings \
-    && apt-get update -qq \
-    && apt-get install -qq -y \
-        # 1. packaging dependencies
-        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
-        # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep dnsutils git python3.12-venv \
-        # 3. frivolous CLI helpers to make debugging failed archiving easier
-        tree nano iputils-ping \
-        # nano iputils-ping dnsutils htop procps jq yq
-    && rm -rf /var/lib/apt/lists/*
-
-# Install sonic search backend
 COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-RUN (which sonic && sonic --version) | tee -a /VERSION.txt
 
-######### Language Environments ####################################
+COPY --from=archivebox-builder /opt/uv/python /opt/uv/python
+COPY --from=archivebox-builder /venv /venv
+COPY --from=archivebox-builder /app /app
+COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
 
-# Set up Python environment
-# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
-#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
-    # && apt-get update -qq \
-    # && apt-get install -qq -y --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
-    # && rm -rf /var/lib/apt/lists/* \
-    # tell PDM to allow using global system python site packages
-    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
-    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
-    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
-    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
-    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
-    # && python3 -m venv $GLOBAL_VENV \
-    # install global dependencies / python build dependencies in GLOBAL_VENV
-    # && pip install --upgrade pip setuptools wheel \
-    # Save version info
-    # && ( \
-    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
-    #     && which pip && pip --version \
-    #     # && which pdm && pdm --version \
-    #     && echo -e '\n\n' \
-    # ) | tee -a /VERSION.txt
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
+    && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
+    && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
+    && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
+    && ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
+    && ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
+    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
+    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
+    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
+    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
+    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
+    && abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
+    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
+    && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$ABXPKG_LIB_DIR" \
+    && openssl rand -hex 16 > /etc/machine-id \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
+    && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
 
-
-# Set up Node environment from the official platform tarball. This avoids
-# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into
-# emulated Docker builds.
-RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    case "$TARGETARCH" in \
-        amd64) NODE_DIST_ARCH="x64" ;; \
-        arm64) NODE_DIST_ARCH="arm64" ;; \
-        *) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \
-    esac \
-    && NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \
-    && echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \
-    && grep "  ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \
-    && tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \
-    && rm "$NODE_TARBALL" SHASUMS256.txt \
-    # Save version info
-    && ( \
-        which node && node --version \
-        && which npm && npm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-# Set up uv and main app /venv
-RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
-ENV UV_COMPILE_BYTECODE=0 \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv
-WORKDIR "$CODE_DIR"
-# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
-    && uv venv /venv --python ${PYTHON_VERSION}
-ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
-RUN uv pip install setuptools pip \
-    && ( \
-        which python3 && python3 --version \
-        && which uv && uv self version \
-        && uv python find --system && uv python find \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-######### ArchiveBox & Extractor Dependencies ##################################
-
-# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc \
-        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
-        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-    && uv pip install \
-        "python-ldap>=3.4.3" \
-    && apt-get purge -y \
-        python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
-
-
-# Runtime config used by plugin hooks. Plugin binaries and npm packages are
-# installed into LIB_DIR below by archivebox init --install and resolved from
-# LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH.
-ENV PERSONAS_DIR=/data/personas \
-    CHROME_EXTENSIONS_DIR=/opt/archivebox/lib/chrome_extensions \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-######### Build Dependencies ####################################
-
-
-# Install ArchiveBox Python venv dependencies from pyproject.toml.
-RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends build-essential gcc python3-dev \
-    && uv --no-cache sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /var/lib/apt/lists/*
-    # installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies
-
-# Setup ArchiveBox runtime config
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/usr/bin/python3.12 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-WORKDIR "$DATA_DIR"
-RUN openssl rand -hex 16 > /etc/machine-id \
-    && mkdir -p "$DATA_DIR" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" \
-    && mkdir -p "$TMP_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
-    && mkdir -p "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-# Pre-bake plugin-managed runtime dependencies using the same abx-dl installer
-# path users run later, before copying ArchiveBox source so source-only edits do
-# not invalidate the heavy browser/plugin dependency layer.
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
-    && export PERSONAS_DIR="$LIB_DIR/personas" \
-    && export CHROME_EXTENSIONS_DIR="$LIB_DIR/chrome_extensions" \
-    && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
-    && mkdir -p "$LIB_DIR" "$LIB_DIR/chrome_extensions" \
-    && apt-get update -qq \
-    && if [ "$TARGETARCH" = "arm64" ]; then \
-        abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \
-        abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
-    fi \
-    && TIMEOUT=600 PUID=0 PGID=0 abx-dl plugins --install \
-    && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \
-    && mkdir -p /opt/archivebox/lib-layer \
-    && cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \
-    && rm -rf /var/lib/apt/lists/* \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer
-
-RUN rm -rf "$LIB_DIR" \
-    && mv /opt/archivebox/lib-layer "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"
-
-# Install ArchiveBox Python package from the checked-out source.
-WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && pip install \
-        --no-deps \
-        "$CODE_DIR" \
-    && ( \
-        pip show archivebox \
-        && which archivebox \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
-    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
-
-# Initialize an empty image collection without rerunning dependency installs.
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
-    && PUID=0 PGID=0 archivebox init \
-    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
+    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
+    && HOME="$TMP_DIR" archivebox init \
+    && (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
         "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
         "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
         "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
         "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        2>/dev/null || true)
+        "$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
+        "$TMP_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
+        2>/dev/null || true) \
+    && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
 
-# Print version for nice docker finish summary
-RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+RUN chmod +x "$CODE_DIR"/bin/*.sh \
+    && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+
+RUN --mount=type=cache,target=/tmp/abxpkg-cache,sharing=locked,mode=1777 \
+    for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
+    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
+    && python3 -c 'from abx_dl.models import discover_plugins; [print(f"export {plugin.enabled_key}=True") for plugin in discover_plugins(runtime="archivebox").values() if plugin.enabled_key in plugin.config.properties]' > /tmp/archivebox-enable-plugins.env \
+    && sort /tmp/archivebox-enable-plugins.env | tee -a /VERSION.txt \
+    && source /tmp/archivebox-enable-plugins.env \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups env HOME=/tmp/abxpkg-cache ABXPKG_NO_CACHE=True ABXPKG_TMP_CACHE_DIR=/tmp/abxpkg-cache archivebox install \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups env HOME=/tmp/abxpkg-cache archivebox version 2>&1 | tee -a /VERSION.txt \
+    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+
+RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
     && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
     ) | tee -a /VERSION.txt
 
-# Verify ArchiveBox is installed and write full version/dependency info.
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
-    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install 2>&1 | tee -a /VERSION.txt \
-    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-####################################################
-
-# Expose ArchiveBox's main interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000

Dockerfile.multistage

@@ -1,229 +0,0 @@
-# syntax=docker/dockerfile:1.7
-
-# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
-# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
-# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
-# Build abx-dl first, then point this file at it:
-#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
-#       --build-context abxbus=../abxbus \
-#       --build-context abxpkg=../abxpkg \
-#       --build-context abx-plugins=../abx-plugins \
-#       -t archivebox/abx-dl:dev
-#   docker buildx build . -f Dockerfile.multistage \
-#       --build-context abx-dl=docker-image://archivebox/abx-dl:dev \
-#       --build-arg ABX_DL_IMAGE=abx-dl \
-#       -t archivebox:multistage
-
-ARG TARGETPLATFORM=linux/amd64
-ARG TARGETOS=linux
-ARG TARGETARCH=amd64
-ARG TARGETVARIANT=
-ARG ABX_DL_IMAGE=abx-dl
-
-FROM ${ABX_DL_IMAGE} AS abx-dl
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ubuntu:24.04 AS archivebox-runtime-base
-
-ARG TARGETPLATFORM=linux/amd64
-ARG TARGETOS=linux
-ARG TARGETARCH=amd64
-ARG TARGETVARIANT
-
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    npm_config_loglevel=error
-
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22.22.3
-
-ENV ARCHIVEBOX_USER=archivebox \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
-
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
-    LIB_BIN_DIR=/opt/archivebox/lib/bin \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
-    PERSONAS_DIR=/data/personas \
-    CHROME_EXTENSIONS_DIR=/opt/archivebox/lib/chrome_extensions \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/venv/bin/python3 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-ENV UV_COMPILE_BYTECODE=0 \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv \
-    VIRTUAL_ENV=/venv \
-    PATH="/venv/bin:/opt/node/bin:$PATH"
-
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
-WORKDIR "$CODE_DIR"
-
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing ArchiveBox base runtime dependencies for $TARGETPLATFORM..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y \
-        apt-transport-https apt-utils ca-certificates curl wget gnupg2 \
-        dumb-init gosu unzip git grep ripgrep dnsutils iputils-ping procps tree nano \
-        cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \
-        libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \
-        libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \
-        libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 \
-        libxext6 libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1 \
-        fonts-liberation fonts-noto-color-emoji xdg-utils \
-        ffmpeg imagemagick tesseract-ocr openjdk-21-jre-headless \
-    && rm -rf /var/lib/apt/lists/*
-
-# Runtime-owned layers copied from the abx-dl image.
-COPY --from=abx-dl /bin/uv /bin/uv
-COPY --from=abx-dl /opt/uv/python /opt/uv/python
-COPY --from=abx-dl /opt/node /opt/node
-COPY --from=abx-dl /venv /venv
-COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt
-
-RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && which python3 && python3 --version \
-    && which node && node --version \
-    && which uv && uv self version \
-    ) | tee -a /VERSION.txt
-
-ENV PYTHONDONTWRITEBYTECODE=1
-
-FROM archivebox-runtime-base AS archivebox-builder
-
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && uv sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && apt-get autoremove -y \
-    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /var/lib/apt/lists/*
-
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && uv pip install --no-deps "$CODE_DIR" \
-    && (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
-
-FROM archivebox-runtime-base
-
-LABEL name="archivebox" \
-    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
-    description="All-in-one self-hosted internet archiving solution" \
-    homepage="https://github.com/ArchiveBox/ArchiveBox" \
-    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
-    org.opencontainers.image.title="ArchiveBox" \
-    org.opencontainers.image.vendor="ArchiveBox" \
-    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
-    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile.multistage"
-
-COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
-COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-
-COPY --from=archivebox-builder /venv /venv
-COPY --from=archivebox-builder /app /app
-COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
-COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib
-
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system "$ARCHIVEBOX_USER" \
-    && useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && (which sonic && sonic --version) | tee -a /VERSION.txt \
-    && mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$LIB_BIN_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && openssl rand -hex 16 > /etc/machine-id \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
-    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-WORKDIR "$DATA_DIR"
-RUN echo "[+] Initializing image collection..." \
-    && PUID=0 PGID=0 archivebox init \
-    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
-        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
-        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
-        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
-        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        2>/dev/null || true)
-
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_BIN_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
-    && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
-    && "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
-    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
-    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
-    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
-    && ! command -v gcc \
-    && ! command -v g++ \
-    && ! command -v make \
-    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
-    ) | tee -a /VERSION.txt
-
-WORKDIR "$DATA_DIR"
-VOLUME "$DATA_DIR"
-EXPOSE 8000
-
-HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
-
-ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Nick Sweeting
+Copyright (c) 2026 Nick Sweeting
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -29,7 +29,7 @@ Without active preservation effort, everything on the internet eventually disapp
 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>
 
-> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  
+> ➡️ Get ArchiveBox with `pip install archivebox>=0.9.5rc1` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  
 
 *Once installed, you can interact with it through the: [Browser Extension](https://github.com/ArchiveBox/archivebox-browser-extension), [CLI](#usage), [self-hosted web interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python API](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [filesystem](#static-archive-exporting).*
 
@@ -92,7 +92,7 @@ docker run -it -v $PWD:/data archivebox/archivebox init
 <br/>
 <br/>
 # Option C: Or install it with your preferred pkg manager (see Quickstart below for apt, brew, and more)
-pip install archivebox
+pip install 'archivebox>=0.9.5rc1'
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
 archivebox install
@@ -295,18 +295,18 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
 <ol>
-<li>Download and install the <code>.deb</code> package from the <a href="https://github.com/ArchiveBox/ArchiveBox/releases">latest release</a>.
-<pre lang="bash"><code style="white-space: pre-line"># download the .deb for your architecture (amd64 or arm64)
-ARCH="$(dpkg --print-architecture)"
-VERSION="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'].lstrip('v'))")"
-curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${VERSION}_${ARCH}.deb" -o /tmp/archivebox.deb
-sudo apt install /tmp/archivebox.deb
-archivebox version                         # make sure all dependencies are installed
+<li>Add the third-party ArchiveBox apt repo and install <code>archivebox</code>.
+<pre lang="bash"><code style="white-space: pre-line">echo 'deb [trusted=yes] https://archivebox.github.io/debian-archivebox dev main' | sudo tee /etc/apt/sources.list.d/archivebox.list
+sudo apt update
+sudo apt install archivebox
+archivebox version                         # make sure the package is installed
 </code></pre>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --install
+archivebox init
+sudo archivebox install
+archivebox add 'https://example.com'
 </code></pre>
 <br/>
 </li>
@@ -476,6 +476,22 @@ For more discussion on managed and paid hosting options see here: <a href="https
 ArchiveBox commands can be run in a terminal [directly on your host](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), or via [Docker](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1)/[Docker Compose](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage).  
 <sup>(depending on how you chose to install it above)</sup>
 
+<!--
+```bash
+set -euo pipefail
+__archivebox_docs_home="$(mktemp -d)"
+export HOME="$__archivebox_docs_home"
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init
+archivebox version
+archivebox help
+test -f index.sqlite3
+test -d archive
+rm -rf "$__archivebox_docs_home"
+```
+-->
+<!--pytest.mark.skip(reason='Mixed host/Docker alternatives include interactive Docker commands')-->
 ```bash
 mkdir -p ~/archivebox/data   # create a new data dir anywhere
 cd ~/archivebox/data         # IMPORTANT: cd into the directory
@@ -512,10 +528,12 @@ archivebox persona create --import=chrome personal
 # make sure you have pip-installed ArchiveBox and it's available in your $PATH first  
 <br/>
 # archivebox [subcommand] [--help]
-archivebox init --install      # safe to run init multiple times (also how you update versions)
+mkdir -p ~/archivebox/data && cd ~/archivebox/data
+archivebox init
+sudo archivebox install
+archivebox add 'https://example.com'
 archivebox version           # get archivebox version info + check dependencies
 archivebox help              # get list of archivebox subcommands that can be run
-archivebox add --depth=1 'https://news.ycombinator.com'
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage">Usage: CLI Usage</a> wiki. ➡️</i>
 </details>
@@ -529,10 +547,11 @@ archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
 # docker compose run archivebox [subcommand] [--help]
-docker compose run archivebox init --install
+docker compose run archivebox init
+docker compose run archivebox install
 docker compose run archivebox version
 docker compose run archivebox help
-docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
+docker compose run archivebox add 'https://example.com'
 # to start webserver: docker compose up
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage">Usage: Docker Compose CLI</a> wiki. ➡️</i>
@@ -547,10 +566,11 @@ docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you create and cd into in a new empty directory first  
 <br/>
 # docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox init --install
+docker run -v $PWD:/data -it archivebox/archivebox init
+docker run -v $PWD:/data -it archivebox/archivebox install
 docker run -v $PWD:/data -it archivebox/archivebox version
 docker run -v $PWD:/data -it archivebox/archivebox help
-docker run -v $PWD:/data -it archivebox/archivebox add --depth=1 'https://news.ycombinator.com'
+docker run -v $PWD:/data -it archivebox/archivebox add 'https://example.com'
 # to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
@@ -676,10 +696,36 @@ docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://exampl
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e1e5bd78-b0b6-45dc-914c-e1046fee4bc4" width="330px" align="right" style="float: right"/>
 
 
+<!--
+```bash
+set -euo pipefail
+__archivebox_docs_home="$(mktemp -d)"
+export HOME="$__archivebox_docs_home"
+mkdir -p ~/archivebox/data ~/Downloads
+cd ~/archivebox/data
+archivebox init
+cat > ~/Downloads/some_feed.xml <<'EOF'
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>ArchiveBox docs test feed</title>
+    <link>https://example.com/</link>
+    <description>ArchiveBox docs test feed</description>
+    <item>
+      <title>Feed item</title>
+      <link>https://example.com/from-feed</link>
+      <guid>https://example.com/from-feed</guid>
+    </item>
+  </channel>
+</rss>
+EOF
+```
+-->
+<!--pytest-codeblocks:cont-->
 ```bash
 # archivebox add --help
 archivebox add 'https://example.com/some/page'
-archivebox add --parser=generic_rss < ~/Downloads/some_feed.xml
+archivebox add --depth=1 --plugins=parse_rss_urls "file://$HOME/Downloads/some_feed.xml"
 archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
 echo 'http://example.com' | archivebox add
 echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add
@@ -689,6 +735,19 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
 # if using Docker Compose, add -T when piping stdin / stdout:
 # echo 'https://example.com' | docker compose run -T archivebox add
 ```
+<!--pytest-codeblocks:cont-->
+<!--
+```bash
+archivebox list --json > snapshots.json
+grep -q 'https://example.com/some/page' snapshots.json
+grep -q 'https://example.com/from-feed' snapshots.json
+grep -q 'https://news.ycombinator.com#2020-12-12' snapshots.json
+grep -q 'http://example.com' snapshots.json
+test -d archive
+test "$(find archive -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" -ge 4
+rm -rf "$__archivebox_docs_home"
+```
+-->
 
 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
 
@@ -1085,7 +1144,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
 </li>
 <li><strong>Try to keep the <code>data/index.sqlite3</code> file on local drive (not a network mount)</strong> or SSD for maximum performance, however the <code>data/archive/</code> folder can be on a network mount or slower HDD.</li>
-<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, you may need to set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid"><code>PUID</code> & <code>PGID</code></a> and <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304">disable <code>root_squash</code></a> on your fileshare server.
+<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, make sure the mounted data directory is writable by its intended owner and consider disabling <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304"><code>root_squash</code></a> on your fileshare server.
 </li>
 </ul>
 
@@ -1095,7 +1154,6 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid">Wiki: Configuration (<code>PUID</code> & <code>GUID</code>)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root">Wiki: Security Overview (Do Not Run as Root)</a></li>
 </ul>
 
@@ -1348,6 +1406,7 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc
 
 First make sure you have `uv` installed: https://docs.astral.sh/uv/getting-started/installation/
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 git clone https://github.com/ArchiveBox/monorepo
 cd monorepo
@@ -1369,6 +1428,7 @@ Repos included in monorepo setup:
 
 #### 2. Option A: Install the Python, JS, and system dependencies directly on your machine
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # Install ArchiveBox runtime dependencies
 mkdir -p data && cd data
@@ -1384,6 +1444,7 @@ archivebox server 0.0.0.0:8000
 
 #### 2. Option B: Build the docker container and use that for development instead
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # Optional: develop via docker by mounting the code dir into the container
 # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
@@ -1413,6 +1474,7 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # set up persistent DEBUG=True for all runs
 archivebox config --set DEBUG=True
@@ -1437,6 +1499,7 @@ https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-dj
 
 If you're looking for the latest `dev` Docker image, it's often available pre-built on Docker Hub, simply pull and use `archivebox/archivebox:dev`.
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 docker pull archivebox/archivebox:dev
 docker run archivebox/archivebox:dev version
@@ -1444,9 +1507,10 @@ docker run archivebox/archivebox:dev version
 ```
 
 ##### Build Branch from Source
-  
+
 You can also build and run any branch yourself from source, for example to build & use `dev` locally:
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # docker-compose.yml:
 services:
@@ -1471,6 +1535,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/lint.sh
 ./bin/test.sh
@@ -1484,6 +1549,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # generate the database migrations after changes to models.py
 cd archivebox/
@@ -1548,6 +1614,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/build.sh
 
@@ -1564,6 +1631,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/release.sh
 

archivebox/__init__.py

@@ -79,16 +79,10 @@ def __getattr__(name: str):
         return VERSION
     if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
         from abx_plugins import get_plugins_dir
+        from .config.constants import CONSTANTS
 
         builtin_plugins_dir = Path(get_plugins_dir()).resolve()
-        user_plugins_dir = (
-            Path(
-                os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR")
-                or os.environ.get("USER_PLUGINS_DIR")
-                or os.environ.get("DATA_DIR", os.getcwd()),
-            )
-            / "custom_plugins"
-        )
+        user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
         plugins = {
             "builtin": builtin_plugins_dir,
             "user": user_plugins_dir,

archivebox/__main__.py

@@ -17,4 +17,4 @@ ASCII_LOGO_MINI = r"""
 """
 
 if __name__ == "__main__":
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main(args=sys.argv[1:])

archivebox/api/auth.py

@@ -51,6 +51,27 @@ def auth_using_token(token: str | None, request: HttpRequest | None = None) -> U
     return user
 
 
+def token_from_request(request: HttpRequest) -> str:
+    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
+    auth_header = request.headers.get("Authorization", "")
+    if not token and auth_header.lower().startswith("bearer "):
+        token = auth_header.split(None, 1)[1].strip()
+    return token
+
+
+def authenticated_user_from_request(request: HttpRequest) -> User | None:
+    user = request.user
+    if user.is_authenticated and user.is_active:
+        return user
+
+    token = token_from_request(request)
+    token_user = auth_using_token(token=token, request=request) if token else None
+    if token_user and token_user.is_active:
+        request.user = token_user
+        return token_user
+    return None
+
+
 def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
     """Given a username and password, check if they are valid and return the corresponding user"""
     user: User | None = None

archivebox/api/models.py

@@ -1,7 +1,7 @@
 __package__ = "archivebox.api"
 
 import secrets
-from archivebox.uuid_compat import uuid7
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 
 from django.conf import settings
 from django.db import models
@@ -17,7 +17,7 @@ def generate_secret_token() -> str:
 
 
 class APIToken(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
@@ -41,7 +41,7 @@ class APIToken(models.Model):
 
 
 class OutboundWebhook(WebhookBase):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)

archivebox/api/urls.py

@@ -7,7 +7,7 @@ from django.shortcuts import redirect
 from django.urls import path
 from django.views.generic.base import RedirectView
 
-from archivebox.core.host_util import build_web_url
+from archivebox.core.routes_util import build_web_url
 from .v1_api import urls as v1_api_urls
 
 

archivebox/api/v1_api.py

@@ -68,14 +68,14 @@ class NinjaAPIWithIOCapture(NinjaAPI):
         # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
 
         # Add Auth Headers to response
-        api_token_attr = getattr(request, "_api_token", None)
+        api_token_attr = request.__dict__.get("_api_token")
         api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
         token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
 
-        response["X-ArchiveBox-Auth-Method"] = str(getattr(request, "_api_auth_method", "None"))
+        response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
         response["X-ArchiveBox-Auth-Expires"] = token_expiry
         response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
-        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if getattr(request.user, "pk", None) else "None"
+        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
         response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
 
         # import ipdb; ipdb.set_trace()

archivebox/api/v1_cli.py

@@ -8,8 +8,11 @@ from enum import Enum
 from django.http import HttpRequest
 
 from ninja import Router, Schema
+from ninja.errors import HttpError
+from pydantic import Field
 
 from archivebox.misc.util import ansi_to_html
+from archivebox.core.models import SnapshotQuerySet
 
 
 # from .auth import API_AUTH_METHODS
@@ -21,6 +24,7 @@ router = Router(tags=["ArchiveBox CLI Sub-Commands"])
 # Schemas
 
 JSONType = list[Any] | dict[str, Any] | bool | int | str | None
+FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]
 
 
 class CLICommandResponseSchema(Schema):
@@ -32,26 +36,11 @@ class CLICommandResponseSchema(Schema):
     stderr: str
 
 
-class FilterTypeChoices(str, Enum):
-    exact = "exact"
-    substring = "substring"
-    regex = "regex"
-    domain = "domain"
-    tag = "tag"
-    timestamp = "timestamp"
-
-
-class StatusChoices(str, Enum):
-    indexed = "indexed"
-    archived = "archived"
-    unarchived = "unarchived"
-    present = "present"
-    valid = "valid"
-    invalid = "invalid"
-    duplicate = "duplicate"
-    orphaned = "orphaned"
-    corrupted = "corrupted"
-    unrecognized = "unrecognized"
+FilterTypeChoices = Enum(
+    "FilterTypeChoices",
+    {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
+    type=str,
+)
 
 
 class AddCommandSchema(Schema):
@@ -66,17 +55,32 @@ class AddCommandSchema(Schema):
     parser: str = "auto"
     plugins: str = ""
     only_new: bool | None = None
+    update: bool = False
+    overwrite: bool = False
     index_only: bool = False
 
 
-class UpdateCommandSchema(Schema):
-    resume: str | None = None
+class SnapshotFilterCommandSchema(Schema):
     after: float | None = 0
-    before: float | None = 999999999999999
+    before: float | None = None
     filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = ["https://example.com"]
+    filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
+    status: str | None = None
+    url__icontains: str | None = None
+    url__istartswith: str | None = None
+    tag: str | None = None
+    crawl_id: str | None = None
+    limit: int | None = None
+    sort: str | None = None
+    search: str | None = None
+
+
+class UpdateCommandSchema(SnapshotFilterCommandSchema):
+    resume: str | None = None
     batch_size: int = 100
     continuous: bool = False
+    index_only: bool = False
+    migrate_only: bool = False
 
 
 class ScheduleCommandSchema(Schema):
@@ -90,38 +94,48 @@ class ScheduleCommandSchema(Schema):
     tag: str = ""
     depth: int = 0
     only_new: bool | None = None
+    update: bool = False
+    overwrite: bool = False
     clear: bool = False
 
 
-class ListCommandSchema(Schema):
-    filter_patterns: list[str] | None = ["https://example.com"]
-    filter_type: str = FilterTypeChoices.substring
-    status: StatusChoices = StatusChoices.indexed
-    after: float | None = 0
-    before: float | None = 999999999999999
-    sort: str = "bookmarked_at"
+class ListCommandSchema(SnapshotFilterCommandSchema):
     as_json: bool = True
     as_html: bool = False
     as_csv: str | None = "timestamp,url"
     with_headers: bool = False
 
 
-class RemoveCommandSchema(Schema):
-    after: float | None = 0
-    before: float | None = 999999999999999
+class RemoveCommandSchema(SnapshotFilterCommandSchema):
     filter_type: str = FilterTypeChoices.exact
-    filter_patterns: list[str] | None = ["https://example.com"]
+    timeout: float = 60.0
+
+
+def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
+    kwargs = args.dict()
+    kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
+    kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
+    return kwargs
 
 
 @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
 def cli_add(request: HttpRequest, args: AddCommandSchema):
     from archivebox.cli.archivebox_add import add
+    from archivebox.misc.util import validate_url
 
     config_overrides: dict[str, object] = {}
     if args.only_new is not None:
         config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
+    submitted_urls: str | list[str] = args.urls
+    if len(args.urls) == 1:
+        try:
+            validate_url(args.urls[0])
+        except ValueError:
+            submitted_urls = args.urls[0]
     crawl, snapshots = add(
-        urls=args.urls,
+        urls=submitted_urls,
         snapshot_ids=args.snapshot_ids,
         tag=args.tag,
         depth=args.depth,
@@ -144,8 +158,8 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         "snapshot_ids": snapshot_ids,
         "queued_urls": args.urls,
     }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
 
     return {
         "success": True,
@@ -159,24 +173,39 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
 
 @router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
 def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import update
+    from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
+    from archivebox.core.snapshot_status import normalize_snapshot_status
 
-    result = update(
-        filter_patterns=args.filter_patterns or [],
-        filter_type=args.filter_type or FilterTypeChoices.substring,
-        after=args.after,
-        before=args.before,
-        resume=args.resume,
-        batch_size=args.batch_size,
-        continuous=args.continuous,
-        stop_daemon_stack=False,
+    try:
+        status = normalize_snapshot_status(args.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+
+    update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    update_kwargs["status"] = status
+    update_kwargs["stop_daemon_stack"] = False
+
+    is_filtered_update = any(
+        (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
     )
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    matched_snapshot_ids = []
+    if is_filtered_update:
+        matched_snapshot_ids = [
+            str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
+        ]
+
+    update(**update_kwargs)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
-        "result": result,
+        "result": {
+            "matched_count": len(matched_snapshot_ids),
+            "snapshot_ids": matched_snapshot_ids,
+        }
+        if is_filtered_update
+        else None,
         "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
         "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
     }
@@ -189,6 +218,8 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
     config_overrides: dict[str, object] = {}
     if args.only_new is not None:
         config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
     result = schedule(
         import_path=args.import_path,
         add=args.add,
@@ -203,8 +234,8 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
         config=config_overrides or None,
     )
 
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
@@ -217,32 +248,38 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
 
 @router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
 def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_search import search
+    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
-    result = search(
-        filter_patterns=args.filter_patterns,
-        filter_type=args.filter_type,
-        status=args.status,
-        after=args.after,
-        before=args.before,
-        sort=args.sort,
-        csv=args.as_csv,
-        json=args.as_json,
-        html=args.as_html,
-        with_headers=args.with_headers,
-    )
+    search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    as_json = search_kwargs.pop("as_json")
+    as_html = search_kwargs.pop("as_html")
+    as_csv = search_kwargs.pop("as_csv")
+    with_headers = search_kwargs.pop("with_headers")
+    try:
+        snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
 
     result_format = "txt"
-    if args.as_json:
+    if as_json:
         result_format = "json"
-        result = json.loads(result)
-    elif args.as_html:
+        result = [
+            json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
+            for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
+        ]
+    elif as_html:
         result_format = "html"
-    elif args.as_csv:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
+    elif as_csv:
         result_format = "csv"
+        cols = [col.strip() for col in as_csv.split(",") if col.strip()]
+        rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
+        result = "\n".join((",".join(cols), *rows) if with_headers else rows)
+    else:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
 
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
@@ -256,37 +293,26 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
 @router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
 def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
     from archivebox.cli.archivebox_remove import remove
-    from archivebox.cli.archivebox_search import get_snapshots
     from archivebox.core.models import Snapshot
 
-    filter_patterns = args.filter_patterns or []
-    snapshots_to_remove = get_snapshots(
-        filter_patterns=filter_patterns,
-        filter_type=args.filter_type,
-        after=args.after,
-        before=args.before,
-    )
-    removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)]
+    remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
+    timeout_arg = remove_kwargs.pop("timeout")
+    timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
+    try:
+        snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
 
-    remove(
+    result = remove(
         yes=True,  # no way to interactively ask for confirmation via API, so we force yes
         snapshots=snapshots_to_remove,
-        before=args.before,
-        after=args.after,
-        filter_type=args.filter_type,
-        filter_patterns=filter_patterns,
+        timeout=timeout,
     )
-
-    result = {
-        "removed_count": len(removed_snapshot_ids),
-        "removed_snapshot_ids": removed_snapshot_ids,
-        "remaining_snapshots": Snapshot.objects.count(),
-    }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
-        "success": True,
-        "errors": [],
+        "success": bool(result["success"]),
+        "errors": [str(result["error"])] if result["error"] else [],
         "result": result,
         "result_format": "json",
         "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",

archivebox/api/v1_core.py

@@ -4,6 +4,7 @@ import math
 import json
 import mimetypes
 import re
+from html import unescape
 from collections import defaultdict
 from pathlib import Path, PurePosixPath
 from uuid import UUID
@@ -29,9 +30,9 @@ from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot, ArchiveResult, Tag
 from archivebox.core.permissions import public_snapshots_queryset
-from archivebox.api.auth import auth_using_token
+from archivebox.api.auth import authenticated_user_from_request
 from archivebox.config.common import get_config
-from archivebox.core.host_util import build_web_url
+from archivebox.core.routes_util import build_web_url
 from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
 from archivebox.core.tag_util import (
     add_snapshot_counts,
@@ -49,7 +50,10 @@ from archivebox.core.tag_util import (
     rename_tag as rename_tag_record,
 )
 from archivebox.crawls.models import Crawl
-from archivebox.api.v1_crawls import CrawlSchema
+from archivebox.api.v1_crawls import CrawlSchema, get_crawl_by_ref
+from archivebox.search.config import get_search_mode, get_search_mode_backend
+from archivebox.search.query import apply_snapshot_search
+from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status
 
 
 router = Router(tags=["Core Models"])
@@ -77,7 +81,7 @@ class CustomPagination(PaginationBase):
     def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
         limit = min(pagination.limit, 500)
         offset = pagination.offset or (pagination.page * limit)
-        total = queryset.count()
+        total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count()
         total_pages = math.ceil(total / limit)
         current_page = math.ceil(offset / (limit + 1))
         items = queryset[offset : offset + limit]
@@ -210,7 +214,10 @@ class ArchiveResultFilterSchema(FilterSchema):
 @paginate(CustomPagination)
 def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
     """List all ArchiveResult entries matching these filters."""
-    return filters.filter(ArchiveResult.objects.all()).distinct()
+    queryset = filters.filter(ArchiveResult.objects.all())
+    if filters.search or filters.snapshot_tag:
+        return queryset.distinct()
+    return queryset
 
 
 def _uuid_ref_query(field_name: str, ref: str) -> Q:
@@ -271,7 +278,7 @@ def _parse_archiveresult_output_json(output_json: str | None) -> dict[str, Any]
 
 
 def _get_archiveresult_upload_data(request: HttpRequest):
-    cached = getattr(request, "_archiveresult_upload_data", None)
+    cached = request.__dict__.get("_archiveresult_upload_data")
     if cached is not None:
         return cached
 
@@ -466,12 +473,7 @@ def _write_archiveresult_files(
             )
 
         guessed_mime = mimetypes.guess_type(relative_output_path)[0]
-        output_mime_type = (
-            (mime_types[0] if mime_types else "")
-            or getattr(uploaded_file, "content_type", None)
-            or guessed_mime
-            or "application/octet-stream"
-        )
+        output_mime_type = (mime_types[0] if mime_types else "") or uploaded_file.content_type or guessed_mime or "application/octet-stream"
         output_files[relative_output_path] = {
             "extension": PurePosixPath(relative_output_path).suffix.lower().lstrip("."),
             "mimetype": output_mime_type,
@@ -499,7 +501,7 @@ def _write_archiveresult_files(
         guessed_mime = mimetypes.guess_type(saved_output_path)[0]
         output_mime_type = (
             (mime_types[index] if index < len(mime_types) else "")
-            or getattr(uploaded_file, "content_type", None)
+            or uploaded_file.content_type
             or guessed_mime
             or "application/octet-stream"
         )
@@ -700,7 +702,7 @@ class SnapshotSchema(Schema):
 
     @staticmethod
     def resolve_archiveresults(obj, context):
-        if bool(getattr(context["request"], "with_archiveresults", False)):
+        if bool(context["request"].__dict__.get("with_archiveresults", False)):
             return obj.archiveresult_set.all().distinct()
         return ArchiveResult.objects.none()
 
@@ -831,7 +833,7 @@ def _snapshots_rss_response(
         crawl_user = snapshot.crawl.created_by if snapshot.crawl_id else None
         description = f"Original URL: {snapshot.url}\nArchived snapshot: {archived_url}"
         feed.add_item(
-            title=snapshot.title or snapshot.url,
+            title=unescape(snapshot.title or snapshot.url),
             link=archived_url or web_base_url,
             description=description,
             unique_id=str(snapshot.id),
@@ -855,12 +857,9 @@ class SnapshotFilterSchema(FilterSchema):
     modified_at: Annotated[datetime | None, FilterLookup("modified_at")] = None
     modified_at__gte: Annotated[datetime | None, FilterLookup("modified_at__gte")] = None
     modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
-    search: Annotated[
-        str | None,
-        FilterLookup(
-            ["url__icontains", "title__icontains", "tags__name__icontains", "id__istartswith", "id__iendswith", "timestamp__startswith"],
-        ),
-    ] = None
+    search: str | None = None
+    search_mode: str | None = None
+    status: str | None = None
     url: Annotated[str | None, FilterLookup("url")] = None
     tag: Annotated[str | None, FilterLookup("tags__name")] = None
     title: Annotated[str | None, FilterLookup("title__icontains")] = None
@@ -868,14 +867,44 @@ class SnapshotFilterSchema(FilterSchema):
     bookmarked_at__gte: Annotated[datetime | None, FilterLookup("bookmarked_at__gte")] = None
     bookmarked_at__lt: Annotated[datetime | None, FilterLookup("bookmarked_at__lt")] = None
 
+    def filter_search(self, value: str | None) -> Q:
+        return Q()
+
+    def filter_search_mode(self, value: str | None) -> Q:
+        return Q()
+
+    def filter_status(self, value: str | None) -> Q:
+        return Q()
+
 
 @router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
 @paginate(CustomPagination)
 def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
     """List all Snapshot entries matching these filters."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.all()
-    return filters.filter(queryset).distinct()
+    try:
+        queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+    queryset = filters.filter(queryset).distinct()
+    query = (filters.search or "").strip()
+    if not query:
+        return queryset
+
+    runtime_config = request.archivebox_config
+    search_mode = get_search_mode(filters.search_mode, config=runtime_config)
+    try:
+        return apply_snapshot_search(
+            queryset,
+            query,
+            search_mode=search_mode,
+            config=runtime_config,
+            include_id_matches=True,
+        )
+    except Exception:
+        if get_search_mode_backend(search_mode, config=runtime_config):
+            return queryset.none()
+        return apply_snapshot_search(queryset, query, search_mode="meta", config=runtime_config, include_id_matches=True)
 
 
 @router.get("/snapshots.rss", url_name="get_snapshots_rss")
@@ -900,18 +929,16 @@ def get_snapshots_rss(
 def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
     """Get a specific Snapshot by id."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.all()
-    try:
-        return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        return queryset.get(_uuid_ref_query("id", snapshot_id))
+    return _get_snapshot_by_ref(snapshot_id)
 
 
 @router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
 def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     tags = normalize_tag_list(data.tags)
-    if data.status is not None and data.status not in Snapshot.StatusChoices.values:
-        raise HttpError(400, f"Invalid status: {data.status}")
+    try:
+        status = normalize_snapshot_status(data.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
     if not data.url.strip():
         raise HttpError(400, "URL is required")
     try:
@@ -922,7 +949,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
         raise HttpError(400, "depth must be between 0 and 4")
 
     if data.crawl_id:
-        crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), data.crawl_id).get()
+        crawl = get_crawl_by_ref(data.crawl_id)
         crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
         tags = tags or crawl_tags
     else:
@@ -939,7 +966,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
         "depth": data.depth,
         "title": data.title,
         "timestamp": str(timezone.now().timestamp()),
-        "status": data.status or Snapshot.StatusChoices.QUEUED,
+        "status": status or Snapshot.StatusChoices.QUEUED,
         "retry_at": timezone.now(),
     }
     snapshot, _ = Snapshot.objects.get_or_create(
@@ -952,10 +979,8 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     if data.title is not None and snapshot.title != data.title:
         snapshot.title = data.title
         update_fields.append("title")
-    if data.status is not None and snapshot.status != data.status:
-        if data.status not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {data.status}")
-        snapshot.status = data.status
+    if status is not None and snapshot.status != status:
+        snapshot.status = status
         update_fields.append("status")
     if update_fields:
         update_fields.append("modified_at")
@@ -976,10 +1001,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
 @router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
 def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
     """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
-    try:
-        snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        snapshot = filter_queryset_by_uuid_substring(Snapshot.objects.all(), snapshot_id).get()
+    snapshot = _get_snapshot_by_ref(snapshot_id)
 
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
@@ -1002,9 +1024,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
         raise HttpError(400, f"Invalid action: {action}")
 
     if "status" in payload:
-        if payload["status"] not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        snapshot.status = payload["status"]
+        try:
+            snapshot.status = normalize_snapshot_status(payload["status"])
+        except ValueError as err:
+            raise HttpError(400, str(err)) from err
         if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
             snapshot.retry_at = None
         update_fields.append("status")
@@ -1060,7 +1083,7 @@ class TagSchema(Schema):
     def resolve_created_by_username(obj):
         user_model = get_user_model()
         user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
+        username = user.username
         return username if isinstance(username, str) else str(user)
 
     @staticmethod
@@ -1069,7 +1092,7 @@ class TagSchema(Schema):
 
     @staticmethod
     def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
+        if bool(context["request"].__dict__.get("with_snapshots", False)):
             return obj.snapshot_set.all().distinct()
         return Snapshot.objects.none()
 
@@ -1212,7 +1235,7 @@ def _get_snapshot_for_tag_edit(snapshot_ref: str) -> Snapshot:
     is_full_uuid = len(snapshot_ref.replace("-", "")) == 32 and all(char in "0123456789abcdef-" for char in snapshot_ref)
     if is_full_uuid:
         try:
-            return snapshot_qs.get(pk=snapshot_ref)
+            return snapshot_qs.get(pk=snapshot_ref.replace("-", ""))
         except (Snapshot.DoesNotExist, ValueError):
             pass
 
@@ -1273,16 +1296,7 @@ def _public_tag_listing_enabled() -> bool:
 
 
 def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
-    user = getattr(request, "user", None)
-    if getattr(user, "is_authenticated", False):
-        return True
-
-    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-    auth_header = request.headers.get("Authorization", "")
-    if not token and auth_header.lower().startswith("bearer "):
-        token = auth_header.split(None, 1)[1].strip()
-
-    if token and auth_using_token(token=token, request=request):
+    if authenticated_user_from_request(request):
         return True
 
     return _public_tag_listing_enabled()
@@ -1294,8 +1308,8 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
     if not _request_has_tag_autocomplete_access(request):
         raise HttpError(401, "Authentication required")
 
-    public_only = not getattr(request.user, "is_authenticated", False) and not getattr(request, "_api_token", None)
-    queryset = get_matching_tags(q, with_snapshot_counts=False)
+    public_only = not request.user.is_authenticated and not request.__dict__.get("_api_token")
+    queryset = get_matching_tags(q)
     public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
     if public_only:
         queryset = queryset.filter(snapshot_set__id__in=public_snapshots.values("id")).distinct()
@@ -1303,7 +1317,7 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
     add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)
 
     return {
-        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": getattr(tag, "num_snapshots", 0)} for tag in tags],
+        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": tag.__dict__.get("num_snapshots", 0)} for tag in tags],
     }
 
 

archivebox/api/v1_crawls.py

@@ -1,5 +1,6 @@
 __package__ = "archivebox.api"
 
+import json
 from pathlib import Path
 from uuid import UUID
 from datetime import datetime
@@ -24,7 +25,7 @@ from archivebox.config.common import get_config
 from archivebox.crawls.models import Crawl
 from archivebox.misc.util import filter_queryset_by_uuid_substring
 
-from .auth import API_AUTH_METHODS, auth_using_token
+from .auth import API_AUTH_METHODS, authenticated_user_from_request
 
 router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
 
@@ -58,7 +59,7 @@ class CrawlSchema(Schema):
     def resolve_created_by_username(obj):
         user_model = get_user_model()
         user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
+        username = user.username
         return username if isinstance(username, str) else str(user)
 
     @staticmethod
@@ -71,7 +72,7 @@ class CrawlSchema(Schema):
 
     @staticmethod
     def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
+        if bool(context["request"].__dict__.get("with_snapshots", False)):
             return obj.snapshot_set.all().distinct()
         return Snapshot.objects.none()
 
@@ -122,9 +123,9 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
 
     tags = normalize_tag_list(data.tags, data.tags_str)
     config = dict(data.config or {})
-    config.setdefault("PERMISSIONS", str(get_config(user=request.user).PERMISSIONS))
+    config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
     crawl = Crawl.objects.create(
-        urls="\n".join(urls),
+        urls="\n".join(json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in urls),
         max_depth=data.max_depth,
         tags_str=",".join(tags),
         label=data.label,
@@ -137,12 +138,16 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
     return crawl
 
 
+def get_crawl_by_ref(crawl_id: str):
+    return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+
+
 @router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
 def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
     """Get a specific Crawl by id."""
     setattr(request, "with_snapshots", with_snapshots)
     setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
 
     if crawl and as_rss:
         query = request.GET.copy()
@@ -156,33 +161,18 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn
 def crawl_file(request: HttpRequest, crawl_id: str, path: str):
     # Try to resolve the crawl first; if it doesn't exist, return 404.
     try:
-        crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+        crawl = get_crawl_by_ref(crawl_id)
     except Crawl.DoesNotExist:
         raise HttpError(404, "Crawl not found")
 
-    # Determine the effective viewer: session user takes precedence, otherwise
-    # fall back to an API token passed via ?api_key=, X-ArchiveBox-API-Key, or
-    # Authorization: Bearer ... (so that programmatic clients still work).
-    user = getattr(request, "user", None)
-    is_authenticated = bool(getattr(user, "is_authenticated", False) and getattr(user, "is_active", False))
-    if not is_authenticated:
-        token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-        auth_header = request.headers.get("Authorization", "")
-        if not token and auth_header.lower().startswith("bearer "):
-            token = auth_header.split(None, 1)[1].strip()
-        token_user = auth_using_token(token=token, request=request) if token else None
-        if token_user and token_user.is_active:
-            user = token_user
-            is_authenticated = True
-            # Re-bind so is_admin_user() / ownership checks below see the token user.
-            setattr(request, "user", token_user)
+    user = authenticated_user_from_request(request)
 
     # Gate access using the same model as SnapshotView/can_view_snapshot:
     # admins always pass; owners can see their own crawls; otherwise the crawl
     # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
     if not is_admin_user(request):
         permissions = normalize_permissions(crawl.permissions)
-        is_owner = bool(is_authenticated and getattr(crawl, "created_by_id", None) == getattr(user, "id", None))
+        is_owner = bool(user and crawl.created_by_id == user.id)
         if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
             raise HttpError(404, "Crawl not found")
 
@@ -217,7 +207,7 @@ def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfol
 @router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
 def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
     """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
 
@@ -259,7 +249,7 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
 
 @router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
 def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
     crawl_id_str = str(crawl.id)
     snapshot_count = crawl.snapshot_set.count()
     deleted_count, _ = crawl.delete()

archivebox/api/v1_personas.py

@@ -8,8 +8,10 @@ from uuid import UUID
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.pagination import paginate
 from pydantic import Field
 
+from archivebox.api.v1_core import CustomPagination
 from archivebox.personas.importers import validate_persona_name
 from archivebox.personas.models import Persona
 
@@ -113,6 +115,7 @@ def find_persona(extension_persona_id: str, name: str) -> Persona | None:
 
 
 @router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
+@paginate(CustomPagination)
 def get_personas(request: HttpRequest):
     """List personas available on this ArchiveBox server."""
     return Persona.objects.all().order_by("name")
@@ -136,7 +139,7 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     created = persona is None
     if persona is None:
         persona = Persona(name=name)
-        if getattr(request.user, "is_authenticated", False):
+        if request.user.is_authenticated:
             persona.created_by = request.user
 
     persona.config = {

archivebox/base_models/admin.py

@@ -5,7 +5,7 @@ __package__ = "archivebox.base_models"
 import json
 import uuid
 from collections.abc import Mapping
-from typing import NotRequired, TypedDict
+from typing import NotRequired, TypedDict, cast
 
 from django import forms
 from django.contrib import admin
@@ -76,39 +76,23 @@ class KeyValueWidget(forms.Widget):
     def _get_config_options(self) -> dict[str, ConfigOption]:
         """Get available config options from plugins."""
         try:
-            from archivebox.config.common import ArchiveBoxConfig
-            from archivebox.hooks import discover_plugin_configs
+            from archivebox.config.common import config_field_metadata
 
             options: dict[str, ConfigOption] = {}
-            skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "SNAP_DIR"}
-            for key, field in ArchiveBoxConfig.model_fields.items():
-                if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys:
-                    continue
-                default = field.default
-                try:
-                    json.dumps(default)
-                except TypeError:
-                    default = str(default)
-                options[key] = {
-                    "plugin": "archivebox",
-                    "type": str(field.annotation),
-                    "default": default,
-                    "description": field.description or "",
+            for key, metadata in config_field_metadata().items():
+                option_type = metadata.get("type", "string")
+                option: ConfigOption = {
+                    "plugin": str(metadata.get("plugin", "archivebox")),
+                    "type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
+                    "default": metadata.get("default", ""),
+                    "description": str(metadata.get("description", "")),
                 }
-
-            plugin_configs = discover_plugin_configs()
-            for plugin_name, schema in plugin_configs.items():
-                for key, prop in schema.get("properties", {}).items():
-                    option: ConfigOption = {
-                        "plugin": plugin_name,
-                        "type": prop.get("type", "string"),
-                        "default": prop.get("default", ""),
-                        "description": prop.get("description", ""),
-                    }
+                schema = metadata.get("schema")
+                if isinstance(schema, Mapping):
                     for schema_key in ("enum", "pattern", "minimum", "maximum"):
-                        if schema_key in prop:
-                            option[schema_key] = prop[schema_key]
-                    options[key] = option
+                        if schema_key in schema:
+                            option[schema_key] = schema[schema_key]
+                options[key] = option
             return options
         except Exception:
             return {}
@@ -839,7 +823,7 @@ class ConfigEditorMixin(admin.ModelAdmin):
         """
         from archivebox.config.common import is_sensitive_config_key
 
-        if change and obj.pk and getattr(obj, "config", None) is not None:
+        if change and obj.pk and obj.config is not None:
             try:
                 stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
             except Exception:

archivebox/base_models/models.py

@@ -2,9 +2,11 @@
 
 __package__ = "archivebox.base_models"
 
+import json
 import shutil
+from typing import Any
 
-from archivebox.uuid_compat import uuid7
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
 
 from django.db import models
@@ -18,6 +20,27 @@ from django.conf import settings
 
 from django_stubs_ext.db.models import TypedModelMeta
 
+from archivebox.config import CONSTANTS
+
+
+def normalize_config_json_values(config: Any) -> Any:
+    if not isinstance(config, dict):
+        return config
+
+    normalized = dict(config)
+    for key, value in list(normalized.items()):
+        if not isinstance(value, str) or len(value) < 2:
+            continue
+        if value[:1] != '"' or value[-1:] != '"':
+            continue
+        try:
+            decoded = json.loads(value)
+        except ValueError:
+            continue
+        if isinstance(decoded, str):
+            normalized[key] = decoded
+    return normalized
+
 
 def get_or_create_system_user_pk(username="system"):
     User = get_user_model()
@@ -44,7 +67,7 @@ class AutoDateTimeField(models.DateTimeField):
 
 
 class ModelWithUUID(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
     created_by = models.ForeignKey(
@@ -92,6 +115,10 @@ class ModelWithHealthStats(models.Model):
     class Meta(TypedModelMeta):
         abstract = True
 
+    @property
+    def admin_change_url(self) -> str:
+        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
+
     @property
     def health(self) -> int:
         total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
@@ -116,6 +143,15 @@ class ModelWithConfig(models.Model):
     class Meta(TypedModelMeta):
         abstract = True
 
+    def save(self, *args, **kwargs):
+        normalized_config = normalize_config_json_values(self.config)
+        if normalized_config != self.config:
+            self.config = normalized_config
+            update_fields = kwargs.get("update_fields")
+            if update_fields is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
+        super().save(*args, **kwargs)
+
 
 class ModelWithDeleteAfter(models.Model):
     delete_after_final_statuses: tuple[str, ...] = ()
@@ -135,7 +171,7 @@ class ModelWithDeleteAfter(models.Model):
     def get_delete_after_config_value(self):
         from archivebox.config.common import get_config
 
-        return get_config(include_machine=False).DELETE_AFTER
+        return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER
 
     def set_delete_at_from_config(self, config_value=None) -> bool:
         if self.delete_at is not None:
@@ -223,9 +259,7 @@ class ModelWithOutputDir(ModelWithUUID):
 
     @classmethod
     def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
-        from archivebox.config.common import get_config
-
-        data_dir = get_config().DATA_DIR.resolve()
+        data_dir = CONSTANTS.DATA_DIR.resolve()
         safe_paths = []
         for raw_path in paths:
             path = Path(raw_path)

archivebox/cli/__init__.py

@@ -18,10 +18,11 @@ if "--debug" in sys.argv:
 
 # Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
 # `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
-# the subcommand executes. Strip it from argv here so each subcommand's own click parser
-# never sees it. Ignored for `help` and `init` themselves.
-if "--init" in sys.argv:
-    sys.argv = [arg for arg in sys.argv if arg != "--init"]
+# the subcommand executes. `--quick-init` is the old server spelling. Strip either from
+# argv here so each subcommand's own click parser never sees it. Ignored for `help` and
+# `init` themselves.
+if "--init" in sys.argv or "--quick-init" in sys.argv:
+    sys.argv = [arg for arg in sys.argv if arg not in ("--init", "--quick-init")]
     os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
 
 
@@ -32,6 +33,7 @@ class ArchiveBoxGroup(click.Group):
         "help": "archivebox.cli.archivebox_help.main",
         "version": "archivebox.cli.archivebox_version.main",
         "mcp": "archivebox.cli.archivebox_mcp.main",
+        "oneshot": "archivebox.cli.archivebox_oneshot.main",
     }
     setup_commands = {
         "init": "archivebox.cli.archivebox_init.main",
@@ -66,10 +68,6 @@ class ArchiveBoxGroup(click.Group):
         # Introspection commands
         "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
     }
-    legacy_model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl_compat.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot_compat.main",
-    }
     all_subcommands = {
         **meta_commands,
         **setup_commands,
@@ -81,34 +79,29 @@ class ArchiveBoxGroup(click.Group):
         "import": "add",
         "archive": "add",
     }
-    legacy_model_subcommands = {
-        "crawl": {"create", "list", "update", "delete"},
-        "snapshot": {"create", "list", "update", "delete"},
-    }
 
     @classmethod
     def get_canonical_name(cls, cmd_name):
         return cls.renamed_commands.get(cmd_name, cmd_name)
 
     @classmethod
-    def _should_use_legacy_model_command(cls, cmd_name: str) -> bool:
-        if cmd_name not in cls.legacy_model_commands:
-            return False
+    def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool:
+        wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
+        return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands)
 
-        try:
-            arg_idx = sys.argv.index(cmd_name)
-        except ValueError:
-            return False
+    @classmethod
+    def _setup_django_for_lazy_import(cls, cmd_name: str) -> None:
+        if not cls._needs_django_for_lazy_import(cmd_name):
+            return
 
-        remaining_args = sys.argv[arg_idx + 1 :]
-        if not remaining_args:
-            return False
+        from django.apps import apps
 
-        first_arg = remaining_args[0]
-        if first_arg in ("-h", "--help"):
-            return False
+        if apps.ready:
+            return
 
-        return first_arg not in cls.legacy_model_subcommands[cmd_name]
+        from archivebox.config.django import setup_django
+
+        setup_django()
 
     def get_command(self, ctx, cmd_name):
         # handle renamed commands
@@ -120,11 +113,9 @@ class ArchiveBoxGroup(click.Group):
             cmd_name = new_name
             ctx.invoked_subcommand = cmd_name
 
-        if self._should_use_legacy_model_command(cmd_name):
-            return self._lazy_load(self.legacy_model_commands[cmd_name])
-
         # handle lazy loading of commands
         if cmd_name in self.all_subcommands:
+            self._setup_django_for_lazy_import(cmd_name)
             return self._lazy_load(cmd_name)
 
         # fall-back to using click's default command lookup
@@ -139,9 +130,9 @@ class ArchiveBoxGroup(click.Group):
 
         # print(f'LAZY LOADING {import_path}')
         mod = import_module(modname)
-        func = getattr(mod, funcname)
+        func = vars(mod)[funcname]
 
-        if not hasattr(func, "__doc__"):
+        if func.__doc__ is None:
             raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
 
         # if not isinstance(cmd, click.BaseCommand):
@@ -199,15 +190,12 @@ def cli(ctx, help=False):
                 raise
 
 
-def main(args=None, prog_name=None, stdin=None):
+def main(args=None, prog_name=None):
     # show `docker run archivebox xyz` in help messages if running in docker
     IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
     IS_TTY = sys.stdin.isatty()
     prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")
 
-    # stdin param allows passing input data from caller (used by __main__.py)
-    # currently not used by click-based CLI, but kept for backwards compatibility
-
     previous_unraisablehook = sys.unraisablehook
 
     def ignore_shutdown_unraisable(unraisable):

archivebox/cli/archivebox_add.py

@@ -5,14 +5,15 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox add"
 
 import sys
-import json
 import os
+import json
 from pathlib import Path
 
 from typing import Any, TYPE_CHECKING
 
 import rich_click as click
 
+from archivebox.config import CONSTANTS
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.util import parse_filesize_to_bytes
 
@@ -23,21 +24,28 @@ if TYPE_CHECKING:
     from archivebox.core.models import Snapshot
 
 
-def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
+def _collect_input_urls(args: tuple[str, ...], *, parser: str = "auto") -> list[str]:
     from archivebox.misc.jsonl import read_args_or_stdin
+    from archivebox.misc.util import validate_url
 
     urls: list[str] = []
     for record in read_args_or_stdin(args):
         url = record.get("url")
         if isinstance(url, str) and url:
-            urls.append(url)
+            try:
+                urls.append(validate_url(url))
+            except ValueError as err:
+                raise click.BadParameter(str(err), param_hint="URL") from err
 
         urls_field = record.get("urls")
         if isinstance(urls_field, str):
             for line in urls_field.splitlines():
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    urls.append(line)
+                    try:
+                        urls.append(validate_url(line))
+                    except ValueError as err:
+                        raise click.BadParameter(str(err), param_hint="URL") from err
 
     return urls
 
@@ -82,7 +90,6 @@ def add(
     crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
     crawl_timeout = int(crawl_timeout or 0)
     snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
-    from archivebox import CONSTANTS
     from archivebox.config.permissions import USER, HOSTNAME
     from archivebox.config.common import get_config
 
@@ -113,29 +120,17 @@ def add(
     from archivebox.misc.logging_util import printable_filesize
     from archivebox.misc.system import get_dir_size
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
-    from django.contrib.auth import get_user_model
     from django.utils import timezone
 
     created_by_id = created_by_id or get_or_create_system_user_pk()
-    created_by = get_user_model().objects.filter(pk=created_by_id).first()
     started_at = timezone.now()
 
-    if isinstance(urls, str):
-        url_list = [line.strip() for line in urls.splitlines() if line.strip()]
-    else:
-        url_list = [str(url).strip() for url in urls if str(url).strip()]
-    if snapshot_ids and len(snapshot_ids) != len(url_list):
-        raise ValueError("snapshot_ids length must match urls length")
-
-    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
-    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
-    sources_file.parent.mkdir(parents=True, exist_ok=True)
-    if snapshot_ids:
-        sources_file.write_text(
-            "\n".join(json.dumps({"url": url, "id": snapshot_ids[index], "tags": tag, "depth": 0}) for index, url in enumerate(url_list)),
-        )
-    else:
-        sources_file.write_text("\n".join(url_list))
+    use_internal_input_root = isinstance(urls, str)
+    source_text = (
+        urls
+        if use_internal_input_root
+        else "\n".join(json.dumps({"type": "CrawlSeed", "url": str(url), "depth": 0}, separators=(",", ":")) for url in urls)
+    )
 
     # 2. Create a new Crawl with inline URLs
     cli_args = [*sys.argv]
@@ -145,13 +140,11 @@ def add(
 
     timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
 
-    # Read URLs directly into crawl
-    urls_content = sources_file.read_text()
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
-    persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
+    persona_obj = Persona.get_or_create_named(persona_name)
     persona_obj.ensure_dirs()
-    effective_persona_config = get_config(persona=persona_obj, user=created_by)
+    effective_persona_config = get_config(persona=persona_obj)
 
     crawl_config = {
         "PERMISSIONS": str(effective_persona_config.PERMISSIONS),
@@ -172,14 +165,22 @@ def add(
         **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
     }
     # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
-    # priority — they win over persona/plugin/env defaults and get stamped
-    # directly onto crawl.config so the runtime resolution and admin UI both
-    # reflect them faithfully.
+    # priority for crawl-frozen keys. Runtime-derived execution keys are
+    # stripped by Crawl.save() and rederived when hooks run.
     crawl_config.update(config_overrides)
 
+    # Crawl.urls is the user's submitted source, not a derived work queue for
+    # this add path. Keeping it byte-for-byte readable is what lets API/UI/CLI
+    # callers audit or resume imports without losing RSS/Netscape/JSON metadata
+    # that is not representable as one plain URL per line.
     crawl = Crawl.objects.create(
-        urls=urls_content,
-        max_depth=depth,
+        urls=source_text,
+        # Stdin/import text gets an extra hop because the synthetic
+        # archivebox://internal root lives at depth 0 and parser-discovered
+        # URLs land at depth 1; direct URL args become the depth=0 input
+        # snapshots themselves so --depth=N matches the deepest hop the user
+        # asked for.
+        max_depth=depth + 1 if use_internal_input_root else depth,
         tags_str=tag,
         persona_id=persona_obj.id,
         label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
@@ -198,15 +199,8 @@ def add(
     #    Discovered URLs become child Snapshots (depth+1)
 
     if index_only:
-        # ``--index-only`` means "add the URLs to the index without archiving
-        # them now". That only holds if we actually materialize the Snapshot
-        # rows here — otherwise the CLI returns success with nothing in the
-        # index, which broke ``test_add_url_after_init`` & friends. Create
-        # the Snapshots synchronously (the same step the runner would do)
-        # but skip starting any worker so extractors don't run.
-        crawl.create_snapshots_from_urls()
-        print("[yellow]\\[*] Index-only mode - URLs indexed, runner not started[/yellow]")
-        return crawl, crawl.snapshot_set.all()
+        print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
+        return crawl, crawl.snapshot_set.none()
 
     # 5. Start the crawl runner to process the queue
     #    The runner will:
@@ -220,12 +214,15 @@ def add(
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
+        from archivebox.services.runner import ensure_background_runner
+
+        ensure_background_runner(allow_under_pytest=True)
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
         from archivebox.machine.models import Process
-        from archivebox.core.takeover_util import command_owns_runtime_stack, current_command, standby_until_runtime_stack_needed
-        from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
+        from archivebox.core.takeover_util import command_owns_foreground_runner, current_command, standby_until_foreground_runner_needed
+        from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process
 
         command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
         exit_code = 0
@@ -233,15 +230,27 @@ def add(
             try:
                 with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
                     while True:
-                        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
                         exit_code = run_runner_worker(
                             ["--crawl-id", str(crawl.id)],
                             name=f"worker_runner_add_{os.getpid()}",
                             interactive_interrupts=True,
                         )
+                        crawl.refresh_from_db(fields=["status", "retry_at"])
+                        if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
+                            break
+                        # A shared supervisord can be stopped by another
+                        # foreground owner (for example `archivebox server`
+                        # shutting down) and stop the one-shot add runner while
+                        # this add's crawl is still runnable. Keep the add
+                        # process alive so it can re-enter the normal
+                        # foreground-runner ownership loop and finish its crawl.
+                        supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None
+                        if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone):
+                            continue
                         if exit_code == 0:
                             break
-                        if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+                        if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
                             continue
                         raise SystemExit(exit_code)
             except KeyboardInterrupt:
@@ -257,13 +266,14 @@ def add(
         # Print summary for foreground runs
         try:
             crawl.refresh_from_db()
-            snapshots_count = crawl.snapshot_set.count()
             try:
                 from django.db.models import Count, Sum
 
-                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("archiveresult__output_size"))
-                total_bytes = int(totals["total_bytes"] or 0) if totals["snapshot_count"] else 0
+                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("output_size"))
+                snapshots_count = int(totals["snapshot_count"] or 0)
+                total_bytes = int(totals["total_bytes"] or 0)
             except Exception:
+                snapshots_count = crawl.snapshot_set.count()
                 total_bytes, _, _ = get_dir_size(crawl.output_dir)
             total_size = printable_filesize(total_bytes)
             total_time = timezone.now() - started_at
@@ -284,9 +294,9 @@ def add(
             except Exception:
                 rel_output_str = str(crawl.output_dir)
 
-            from archivebox.core.host_util import build_admin_url
+            from archivebox.core.routes_util import build_admin_url
 
-            admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id.hex}/change/", config=config)
+            admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
 
             print("\n[bold]crawl output saved to:[/bold]")
             print(f"  {rel_output_str}")
@@ -321,6 +331,7 @@ def add(
 @click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
 @click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
 @click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
+@click.option("--extract", default="", help="Comma-separated plugins or output types to extract e.g. title,pdf,text/html,image,...")
 @click.option("--persona", default="Default", help="Authentication profile to use when archiving")
 @click.option(
     "--only-new/--no-only-new",
@@ -330,6 +341,8 @@ def add(
     "Pass --no-only-new to force re-archive of URLs that already exist.",
 )
 @click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
+@click.option("--overwrite", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
+@click.option("--update", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
 @click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
 @click.argument("urls", nargs=-1, type=click.Path())
 @docstring(add.__doc__)
@@ -340,7 +353,12 @@ def main(**kwargs):
 
     with foreground_shutdown_signals(), foreground_parent_watchdog():
         raw_urls = kwargs.pop("urls")
-        urls = _collect_input_urls(raw_urls)
+        if raw_urls:
+            urls = _collect_input_urls(raw_urls, parser=kwargs.get("parser", "auto"))
+        elif not sys.stdin.isatty():
+            urls = sys.stdin.read()
+        else:
+            urls = []
         if not urls:
             raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
         if int(kwargs.get("max_urls") or 0) < 0:
@@ -360,9 +378,26 @@ def main(**kwargs):
 
         # Translate --only-new/--no-only-new into a crawl config override.
         # add() takes config overrides as a dict; no per-flag kwargs.
+        overwrite = kwargs.pop("overwrite", False)
+        update = kwargs.pop("update", False)
         only_new = kwargs.pop("only_new", None)
+        extract = kwargs.pop("extract", "")
+        if overwrite or update:
+            only_new = False
         if only_new is not None:
             kwargs["config"] = {"ONLY_NEW": bool(only_new)}
+        if extract:
+            from abx_dl.models import discover_plugins, plugins_matching_output
+
+            all_plugins = discover_plugins()
+            tokens = [token.strip() for token in extract.split(",") if token.strip()]
+            plugin_names = {name.lower(): name for name in all_plugins}
+            selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
+            selected += plugins_matching_output(all_plugins, tokens)
+            if not selected:
+                raise click.UsageError(f"No plugins found matching extract types: {extract}")
+            existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
+            kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
 
         add(urls=urls, **kwargs)
 

archivebox/cli/archivebox_archiveresult.py

@@ -74,7 +74,7 @@ def create_archiveresults(
         1: Failure
     """
     from archivebox.config.common import get_config
-    from archivebox.hooks import discover_hooks
+    from archivebox.plugins.hooks import discover_hooks
     from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
     from archivebox.core.models import Snapshot
 

archivebox/cli/archivebox_config.py

@@ -6,6 +6,7 @@ import sys
 import toml
 import rich_click as click
 from rich import print
+from pathlib import Path
 
 from archivebox.misc.util import docstring, enforce_types
 from archivebox.misc.toml_util import CustomTOMLEncoder
@@ -30,12 +31,17 @@ def config(
     from archivebox.misc.logging_util import printable_config
     from abx_plugins.plugins.base.utils import resolve_alias
     from archivebox.config.collection import write_config_file
+    from archivebox.config import CONSTANTS_CONFIG
     from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
 
     check_data_folder()
 
     FLAT_CONFIG = get_config().as_dict()
+    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys()
+    readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)}
+    writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config}
+    readable_config = {**writable_config, **readonly_config}
     CONFIGS = get_all_configs()
     plugin_schemas = {
         plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
@@ -56,18 +62,23 @@ def config(
             config_options = [
                 core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
             ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
             for config_section in CONFIGS.values():
-                aliases = getattr(config_section, "aliases", {})
+                aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias}
 
                 for search_key in config_options:
                     # search all aliases in the section
                     for alias_key, key in aliases.items():
-                        if search_key.lower() in alias_key.lower():
+                        if key in readable_config and search_key.lower() in alias_key.lower():
                             matching_config[key] = dict(config_section)[key]
 
                     # search all keys and values in the section
                     for existing_key, value in dict(config_section).items():
+                        if existing_key in readable_config and (
+                            search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower()
+                        ):
+                            matching_config[existing_key] = value
+                    for existing_key, value in readonly_config.items():
                         if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
                             matching_config[existing_key] = value
 
@@ -79,18 +90,18 @@ def config(
             config_options = [
                 core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
             ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
+            failed_config = [key for key in config_options if key not in readable_config]
             if failed_config:
                 print("\n[red][X] These options failed to get[/red]")
                 print("    {}".format("\n    ".join(config_options)))
                 raise SystemExit(1)
         else:
-            matching_config = FLAT_CONFIG
+            matching_config = readable_config
 
         # Display core config sections
         for config_section in CONFIGS.values():
-            section_header = getattr(config_section, "toml_section_header", "")
+            section_header = config_section.toml_section_header
             if isinstance(section_header, str) and section_header:
                 print(f"[grey53]\\[{section_header}][/grey53]")
             else:
@@ -100,12 +111,18 @@ def config(
             print(_format_toml(kv_in_section))
             print("[grey53]################################################################[/grey53]")
 
+        readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config}
+        if readonly_keys:
+            print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
+            print(_format_toml(readonly_keys))
+            print("[grey53]################################################################[/grey53]")
+
         plugin_keys = {}
 
         # Collect all plugin config keys
         for schema in plugin_schemas.values():
             for key in schema.keys():
-                if key in matching_config:
+                if key in matching_config and key in writable_config:
                     plugin_keys[key] = matching_config[key]
 
         # Display all plugin config in single [PLUGINS] section
@@ -135,7 +152,7 @@ def config(
                     f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
                 )
 
-            if key in FLAT_CONFIG:
+            if key in writable_config:
                 new_config[key] = val.strip()
             else:
                 failed_options.append(line)

archivebox/cli/archivebox_crawl.py

@@ -65,6 +65,7 @@ def create_crawl(
         1: Failure
     """
     from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
+    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
 
@@ -98,7 +99,11 @@ def create_crawl(
         # Collect URLs
         url = record.get("url")
         if url:
-            url_list.append(url)
+            try:
+                url_list.append(validate_url(str(url)))
+            except ValueError as err:
+                rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                return 1
 
         # Handle 'urls' field (newline-separated)
         urls_field = record.get("urls")
@@ -106,7 +111,11 @@ def create_crawl(
             for line in urls_field.split("\n"):
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    url_list.append(line)
+                    try:
+                        url_list.append(validate_url(line))
+                    except ValueError as err:
+                        rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                        return 1
 
     # Output pass-through records first
     if not is_tty:

archivebox/cli/archivebox_crawl_compat.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_add import add
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--wait/--no-wait", "wait", default=True, help="Accepted for backwards compatibility")
-@click.argument("urls", nargs=-1)
-def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox crawl URL...` entrypoint."""
-    del status, wait
-    add(list(urls), depth=depth, tag=tag, bg=True)
-    sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_extract.py

@@ -48,10 +48,11 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     from rich import print as rprint
     from django.utils import timezone
     from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.services.runner import run_crawl
 
     try:
-        archiveresult = ArchiveResult.objects.get(id=archiveresult_id)
+        archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
     except ArchiveResult.DoesNotExist:
         rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
         return 1
@@ -116,6 +117,7 @@ def run_plugins(
     wait: bool = True,
     emit_results: bool = True,
     show_progress: bool = True,
+    preserve_queued: bool = False,
 ) -> int:
     """
     Run plugins on Snapshots from input.
@@ -209,7 +211,7 @@ def run_plugins(
             if snapshot_id in existing_snapshot_ids
             for plugin_name in plugin_names
         )
-    plugins_by_name = discover_plugins()
+    plugins_by_name = discover_plugins(runtime="archivebox")
     requested_rows: set[tuple[str, str, str]] = set()
     for snapshot_id, plugin_name in requested_pairs:
         plugin = plugins_by_name.get(plugin_name)
@@ -219,6 +221,18 @@ def run_plugins(
         else:
             requested_rows.add((snapshot_id, plugin_name, ""))
 
+    queued_rows: set[tuple[str, str, str]] = set()
+    if preserve_queued and requested_rows:
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+    rows_to_queue = requested_rows - queued_rows
+
     reset_fields = {
         "status": ArchiveResult.StatusChoices.QUEUED,
         "output_str": "",
@@ -230,23 +244,34 @@ def run_plugins(
         "end_ts": None,
         "modified_at": timezone.now(),
     }
-    if plugins_list:
-        ArchiveResult.objects.filter(snapshot_id__in=existing_snapshot_ids, plugin__in=plugins_list).update(**reset_fields)
-    elif requested_plugins_by_snapshot:
-        snapshot_ids_by_plugin: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, plugin_names in requested_plugins_by_snapshot.items():
-            if snapshot_id in existing_snapshot_ids:
-                for plugin_name in plugin_names:
-                    snapshot_ids_by_plugin[plugin_name].add(snapshot_id)
-        for plugin_name, plugin_snapshot_ids in snapshot_ids_by_plugin.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(**reset_fields)
-    existing_rows = set(
-        ArchiveResult.objects.filter(
-            snapshot_id__in=existing_snapshot_ids,
-            plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
-        ).values_list("snapshot_id", "plugin", "hook_name"),
+    if rows_to_queue and plugins_list:
+        rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    elif rows_to_queue and requested_plugins_by_snapshot:
+        snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    existing_rows = (
+        {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        if rows_to_queue
+        else set()
     )
-    missing_rows = requested_rows - {(str(snapshot_id), plugin_name, hook_name) for snapshot_id, plugin_name, hook_name in existing_rows}
+    missing_rows = rows_to_queue - existing_rows
     if missing_rows:
         ArchiveResult.objects.bulk_create(
             [
@@ -271,7 +296,23 @@ def run_plugins(
             # also keep status=paused here: `retry_at` only asks the orchestrator
             # to process the queued plugin rows, and run_due_snapshot restores
             # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
-            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "modified_at"):
+            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
+            if preserve_queued and queued_rows:
+                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
+                    .filter(retry_at__gt=queue_at)
+                    .values_list("id", flat=True)
+                )
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list(
+                        "id",
+                        flat=True,
+                    )
+                )
+            for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
                 # Guard the read-time status so we never bump retry_at on a
                 # row that's been re-queued / started by a concurrent runner.
                 snapshot.safe_update(
@@ -378,16 +419,11 @@ def run_plugins(
 
 
 def is_archiveresult_id(value: str) -> bool:
-    """Check if value looks like an ArchiveResult UUID."""
-    import re
-
-    uuid_pattern = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
-    if not uuid_pattern.match(value):
-        return False
-    # Verify it's actually an ArchiveResult (not a Snapshot or other object)
+    """Check if value resolves to an ArchiveResult ID."""
     from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
 
-    return ArchiveResult.objects.filter(id=value).exists()
+    return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists()
 
 
 @click.command()

archivebox/cli/archivebox_help.py

@@ -13,6 +13,10 @@ from rich.panel import Panel
 
 
 def _command_doc(cmd: str, import_path: str) -> str:
+    def first_doc_line(docstring: str | None) -> str:
+        lines = (docstring or "").splitlines()
+        return lines[0] if lines else ""
+
     modname, _ = import_path.rsplit(".", 1)
     spec = importlib.util.find_spec(modname)
     if spec is None or spec.origin is None:
@@ -26,9 +30,9 @@ def _command_doc(cmd: str, import_path: str) -> str:
     for name in (cmd, "main"):
         for node in tree.body:
             if isinstance(node, ast.FunctionDef) and node.name == name:
-                return (ast.get_docstring(node) or "").splitlines()[0]
+                return first_doc_line(ast.get_docstring(node))
 
-    return (ast.get_docstring(tree) or "").splitlines()[0]
+    return first_doc_line(ast.get_docstring(tree))
 
 
 def help() -> None:
@@ -85,8 +89,8 @@ def help() -> None:
     [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
 """)
 
-    config = get_config()
-    if os.access(config.ARCHIVE_DIR, os.R_OK) and config.ARCHIVE_DIR.is_dir():
+    get_config()
+    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
         pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
         EXAMPLE_USAGE = f"""
 [light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]

archivebox/cli/archivebox_init.py

@@ -25,7 +25,7 @@ def _display_data_path(path: Path, data_dir: Path) -> str:
 def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
     """Initialize a new ArchiveBox collection in the current directory"""
 
-    from archivebox.config import CONSTANTS, VERSION, DATA_DIR
+    from archivebox.config import CONSTANTS, VERSION
     from archivebox.config.common import get_config
     from archivebox.config.collection import write_config_file
     from archivebox.misc.db import apply_migrations
@@ -37,7 +37,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
     #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
 
-    is_empty = not len(set(os.listdir(DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
+    is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
     existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
     if is_empty and not existing_index:
         print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
@@ -65,21 +65,23 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     else:
         print("\n[green][+] Building archive folder structure...[/green]")
 
-    archive_path = _display_data_path(config.ARCHIVE_DIR, DATA_DIR)
-    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, DATA_DIR)
-    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, DATA_DIR)
+    archive_path = _display_data_path(CONSTANTS.ARCHIVE_DIR, CONSTANTS.DATA_DIR)
+    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, CONSTANTS.DATA_DIR)
+    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, CONSTANTS.DATA_DIR)
     print(f"    + {archive_path}, {sources_path}, {logs_path}...")
     Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
-    config.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
-    config.USERS_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.USERS_DIR.mkdir(parents=True, exist_ok=True)
     Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
+    for path in (Path(CONSTANTS.SOURCES_DIR), CONSTANTS.ARCHIVE_DIR, CONSTANTS.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
+        path.chmod(int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)
 
-    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, DATA_DIR)}...")
+    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, CONSTANTS.DATA_DIR)}...")
 
     # create the .archivebox_id file with a unique ID for this collection
     from archivebox.config.paths import _get_collection_id
 
-    _get_collection_id(DATA_DIR, force_create=True)
+    _get_collection_id(CONSTANTS.DATA_DIR, force_create=True)
 
     # create the ArchiveBox.conf file
     write_config_file({"SECRET_KEY": config.SECRET_KEY})
@@ -92,14 +94,22 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     from archivebox.config.django import setup_django
 
     setup_django()
-    check_migrations(blocking=True, auto_apply=False)
+    previous_wants_init = os.environ.get("ARCHIVEBOX_WANTS_INIT")
+    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+    try:
+        check_migrations(blocking=True, auto_apply=False)
 
-    for migration_line in apply_migrations(DATA_DIR):
-        sys.stdout.write(f"    {migration_line}\n")
+        for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
+            sys.stdout.write(f"    {migration_line}\n")
+    finally:
+        if previous_wants_init is None:
+            os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
+        else:
+            os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
 
     assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
     print()
-    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, DATA_DIR)}")
+    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
 
     # from django.contrib.auth.models import User
     #     call_command("createsuperuser", interactive=True)
@@ -143,8 +153,8 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
 
     config = get_config()
     config.TMP_DIR.mkdir(parents=True, exist_ok=True)
-    config.LIB_DIR.mkdir(parents=True, exist_ok=True)
-    (config.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
+    config.ABXPKG_LIB_DIR.mkdir(parents=True, exist_ok=True)
+    (config.ABXPKG_LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
 
     working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
     if working_tmp_dir:
@@ -153,7 +163,6 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
     if working_lib_dir:
         working_lib_dir.mkdir(parents=True, exist_ok=True)
-        (working_lib_dir / "bin").mkdir(parents=True, exist_ok=True)
 
     if install:
         from archivebox.cli.archivebox_install import install as install_method
@@ -177,7 +186,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
 @click.command()
 @click.option("--force", "-f", is_flag=True, help="Ignore unrecognized files in current directory and initialize anyway")
 @click.option("--quick", "-q", is_flag=True, help="Run any updates or migrations without rechecking all snapshot dirs")
-@click.option("--install", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
+@click.option("--install", "--setup", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
 @docstring(init.__doc__)
 def main(**kwargs) -> None:
     init(**kwargs)

archivebox/cli/archivebox_install.py

@@ -3,6 +3,8 @@
 __package__ = "archivebox.cli"
 
 import os
+import shutil
+from pathlib import Path
 
 import rich_click as click
 from rich import print
@@ -10,6 +12,38 @@ from rich import print
 from archivebox.misc.util import docstring, enforce_types
 
 
+def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:
+    """Expose the shared abxpkg lib dir at DATA_DIR/lib after install."""
+    link_path = data_dir / "lib"
+    target_config_path = abxpkg_lib_dir.expanduser().absolute()
+    target_path = abxpkg_lib_dir.expanduser().resolve()
+    from archivebox.config.paths import get_machine_type
+
+    scoped_data_lib_path = (link_path / get_machine_type()).absolute()
+
+    if target_config_path == scoped_data_lib_path:
+        if link_path.is_symlink() or link_path.is_file():
+            link_path.unlink()
+        target_config_path.mkdir(parents=True, exist_ok=True)
+        return None
+
+    if link_path.resolve(strict=False) == target_path:
+        return None
+
+    target_path.mkdir(parents=True, exist_ok=True)
+
+    if link_path.is_symlink():
+        if link_path.resolve(strict=False) == target_path:
+            return link_path
+        link_path.unlink()
+    elif link_path.exists():
+        shutil.rmtree(link_path)
+
+    relative_target = os.path.relpath(target_path, start=link_path.parent)
+    link_path.symlink_to(relative_target, target_is_directory=True)
+    return link_path
+
+
 @enforce_types
 def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bool = False) -> None:
     """Detect and install ArchiveBox dependencies by running the abx-dl install flow
@@ -22,12 +56,16 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
     """
 
     from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
-    from archivebox.config.common import get_config
+    from archivebox.config import CONSTANTS
     from archivebox.misc.logging import stderr
     from archivebox.cli.archivebox_init import init
 
-    config = get_config()
-    archive_dir = config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
+
+    if dry_run:
+        print("[dim]Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow[/dim]")
+        return
+
     if not (os.access(archive_dir, os.R_OK) and archive_dir.is_dir()):
         init()  # must init full index because we need a db to store Binary entries in
 
@@ -47,10 +85,6 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
         print(f"    DATA_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].")
         print()
 
-    if dry_run:
-        print("[dim]Dry run - would run the abx-dl install flow[/dim]")
-        return
-
     # Set up Django
     from archivebox.config.django import setup_django
 
@@ -67,6 +101,11 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
 
     run_install(plugin_names=plugin_names or None)
 
+    from archivebox.config.common import get_config
+
+    config = get_config(include_machine=False)
+    ensure_data_dir_lib_symlink(CONSTANTS.DATA_DIR, config.ABXPKG_LIB_DIR)
+
     print()
 
     # Check for superuser

archivebox/cli/archivebox_list.py

@@ -7,50 +7,15 @@ import sys
 
 import rich_click as click
 
-from archivebox.cli.archivebox_snapshot import list_snapshots
+from archivebox.cli.archivebox_snapshot import list_snapshots, snapshot_filter_options, snapshot_output_options
 
 
 @click.command()
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def main(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-) -> None:
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def main(**kwargs) -> None:
     """List Snapshots."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_oneshot.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox oneshot"
+
+import subprocess
+from pathlib import Path
+
+import rich_click as click
+
+from archivebox.config import CONSTANTS
+
+
+@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
+@click.argument("args", nargs=-1)
+def main(args: tuple[str, ...] = ()) -> None:
+    """Download URLs using abx-dl"""
+    cwd = Path.cwd()
+    if any((path / CONSTANTS.SQL_INDEX_FILENAME).exists() for path in (cwd, *cwd.parents)):
+        raise click.ClickException(
+            "Refusing to run `archivebox oneshot` inside an ArchiveBox DATA_DIR. Use `archivebox add` here, or run oneshot from another directory.",
+        )
+    raise SystemExit(subprocess.run(["abx-dl", *args]).returncode)
+
+
+if __name__ == "__main__":
+    main()

archivebox/cli/archivebox_persona.py

@@ -372,7 +372,7 @@ def create_personas(
         if not is_tty:
             write_record(
                 {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "id": str(persona.id),
                     "name": persona.name,
                     "path": str(persona.path),
                     "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -424,7 +424,7 @@ def list_personas(
         else:
             write_record(
                 {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "id": str(persona.id),
                     "name": persona.name,
                     "path": str(persona.path),
                     "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -500,7 +500,7 @@ def update_personas(name: str | None = None) -> int:
             if not is_tty:
                 write_record(
                     {
-                        "id": str(persona.id) if hasattr(persona, "id") else None,
+                        "id": str(persona.id),
                         "name": persona.name,
                         "path": str(persona.path),
                     },

archivebox/cli/archivebox_pluginmap.py

@@ -16,8 +16,9 @@ EVENT_FLOW_DIAGRAM = """
 │  InstallEvent                                                               │
 │    └─ config.json > required_binaries                                       │
 │         └─ BinaryRequestEvent                                               │
-│              └─ on_BinaryRequest__*                                         │
+│              └─ abxpkg BinaryService builtin providers                      │
 │                   └─ BinaryEvent                                            │
+│                        └─ BinaryCacheService / project cache backend        │
 │                                                                             │
 │  CrawlEvent                                                                 │
 │    └─ CrawlSetupEvent                                                       │
@@ -56,13 +57,15 @@ def pluginmap(
     from rich.panel import Panel
     from rich import box
 
-    from archivebox.hooks import (
-        BUILTIN_PLUGINS_DIR,
-        USER_PLUGINS_DIR,
+    from archivebox.plugins.hooks import (
         discover_hooks,
         is_background_hook,
         normalize_hook_event_name,
     )
+    from archivebox.plugins.discovery import (
+        BUILTIN_PLUGINS_DIR,
+        USER_PLUGINS_DIR,
+    )
 
     console = Console()
     prnt = console.print
@@ -73,12 +76,14 @@ def pluginmap(
             "emits": ["BinaryRequestEvent", "BinaryEvent", "ProcessEvent"],
         },
         "BinaryRequestEvent": {
-            "description": "Provider phase. on_BinaryRequest hooks resolve or install requested binaries.",
+            "description": "Binary resolution phase. abxpkg BinaryService resolves or installs requested binaries using built-in providers.",
             "emits": ["BinaryEvent", "ProcessEvent"],
+            "direct_hooks": False,
         },
         "BinaryEvent": {
-            "description": "Resolved binary metadata event. Projected into the DB binary cache.",
+            "description": "Resolved binary metadata event. ArchiveBoxBinaryService projects it into the ArchiveBox DB binary cache.",
             "emits": [],
+            "direct_hooks": False,
         },
         "CrawlEvent": {
             "description": "Root crawl lifecycle event emitted by the runner.",
@@ -138,7 +143,8 @@ def pluginmap(
 
     for event_name, info in event_phases.items():
         hook_event = normalize_hook_event_name(event_name)
-        hooks = discover_hooks(event_name, filter_disabled=not show_disabled)
+        has_direct_hooks = info.get("direct_hooks", True)
+        hooks = discover_hooks(event_name, filter_disabled=not show_disabled) if has_direct_hooks else []
 
         hook_infos = []
         for hook_path in hooks:
@@ -192,8 +198,10 @@ def pluginmap(
         prnt(f"[dim]{info['description']}[/dim]")
         if info["emits"]:
             prnt(f"[dim]Emits: {', '.join(info['emits'])}[/dim]")
-        if not hook_infos:
+        if not hook_infos and has_direct_hooks:
             prnt(f"[dim]No direct on_{hook_event}__* scripts are currently defined for this event family.[/dim]")
+        elif not has_direct_hooks:
+            prnt("[dim]No direct plugin hook family. This event is handled by services.[/dim]")
         prnt()
 
     if not quiet:

archivebox/cli/archivebox_remove.py

@@ -12,7 +12,7 @@ import rich_click as click
 from django.db import OperationalError
 from django.db.models import QuerySet
 
-from archivebox.config import DATA_DIR
+from archivebox.config import CONSTANTS
 from archivebox.config.django import setup_django
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.checks import check_data_folder
@@ -23,6 +23,7 @@ from archivebox.misc.logging_util import (
     log_removal_finished,
     TimedProgress,
 )
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
 
 @enforce_types
@@ -33,49 +34,56 @@ def remove(
     after: float | None = None,
     before: float | None = None,
     yes: bool = False,
-    out_dir: Path = DATA_DIR,
-) -> QuerySet:
+    delete: bool = False,
+    out_dir: Path = CONSTANTS.DATA_DIR,
+    timeout: float | None = None,
+    **kwargs,
+) -> dict[str, object]:
     """Remove the specified URLs from the archive"""
 
     setup_django()
     check_data_folder()
+    timeout = float(timeout) if timeout is not None else None
 
-    from archivebox.cli.archivebox_search import get_snapshots
+    from archivebox.core.models import Snapshot
 
+    filter_kwargs = {
+        **kwargs,
+        "filter_patterns": filter_patterns,
+        "filter_type": filter_type,
+        "after": after,
+        "before": before,
+    }
     pattern_list = list(filter_patterns)
 
     log_list_started(pattern_list or None, filter_type)
     timer = TimedProgress(360, prefix="      ")
     try:
-        snapshots = get_snapshots(
-            snapshots=snapshots,
-            filter_patterns=pattern_list or None,
-            filter_type=filter_type,
-            after=after,
-            before=before,
-        )
+        if snapshots is None:
+            snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
+        # Freeze the target set up-front so a concurrent daemon writing new
+        # snapshots can't extend the deletion under us, and so the cursor isn't
+        # held open across the per-row deletes below.
+        snapshot_pks = list(snapshots.values_list("pk", flat=True))
     finally:
         timer.end()
 
-    if not snapshots.exists():
+    if not snapshot_pks:
         log_removal_finished(0, 0)
         raise SystemExit(1)
 
-    log_list_finished(snapshots)
-    log_removal_started(snapshots, yes=yes)
+    if not yes:
+        log_list_finished(snapshots)
+        log_removal_started(snapshots, yes=False)
 
-    from archivebox.core.models import Snapshot
-    from archivebox.search import flush_search_index
+    from archivebox.search.query import flush_search_index
 
-    # Freeze the target set up-front so a concurrent daemon writing new
-    # snapshots can't extend the deletion under us, and so the cursor isn't
-    # held open across the per-row deletes below.
-    snapshot_pks = list(snapshots.values_list("pk", flat=True))
-    to_remove = len(snapshot_pks)
+    started_at = time.monotonic()
+    deadline = started_at + timeout if timeout is not None else None
 
     # Search-index flush touches a separate backend (FTS / sonic), not the
     # main index.sqlite3 writer lock, so it's safe to do once up front.
-    flush_search_index(snapshots=Snapshot.objects.filter(pk__in=snapshot_pks))
+    flush_search_index(snapshots=snapshots)
 
     # Delete one snapshot at a time. Each ``.delete()`` is its own short
     # Django-atomic block, so the writer lock is released between rows and
@@ -90,40 +98,60 @@ def remove(
     # own retry loop at this outer (non-atomic) level. Each attempt is a
     # fresh atomic; an exception cleanly rolls it back before we sleep.
     retry_interval = 1.0
-    retry_timeout = 60.0
-    for pk in snapshot_pks:
-        deadline = time.monotonic() + retry_timeout
+    deleted_snapshot_pks = []
+    timed_out = False
+    timeout_error = ""
+    for index, pk in enumerate(snapshot_pks):
+        if deadline is not None and time.monotonic() >= deadline:
+            timed_out = True
+            timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining."
+            break
         while True:
             try:
-                Snapshot.objects.filter(pk=pk).delete()
+                deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
+                if deleted_count:
+                    deleted_snapshot_pks.append(pk)
                 break
             except OperationalError as err:
-                if "database is locked" not in str(err) or time.monotonic() >= deadline:
+                if "database is locked" not in str(err):
                     raise
-                time.sleep(retry_interval)
+                remaining_time = deadline - time.monotonic() if deadline is not None else None
+                if remaining_time is not None and remaining_time <= 0:
+                    timed_out = True
+                    timeout_error = f"Remove timed out after {timeout:g}s while waiting for the database lock."
+                    break
+                time.sleep(min(retry_interval, remaining_time) if remaining_time is not None else retry_interval)
+        if timed_out:
+            break
 
     all_snapshots = Snapshot.objects.all()
-    log_removal_finished(all_snapshots.count(), to_remove)
+    remaining_count = all_snapshots.count()
+    deleted_snapshot_id_set = set(deleted_snapshot_pks)
+    remaining_snapshot_pks = [snapshot_id for snapshot_id in snapshot_pks if snapshot_id not in deleted_snapshot_id_set]
+    log_removal_finished(remaining_count, len(deleted_snapshot_pks))
 
-    return all_snapshots
+    return {
+        "removed_count": len(deleted_snapshot_pks),
+        "removed_snapshot_ids": [str(snapshot_id) for snapshot_id in deleted_snapshot_pks],
+        "not_removed_count": len(remaining_snapshot_pks),
+        "not_removed_snapshot_ids": [str(snapshot_id) for snapshot_id in remaining_snapshot_pks],
+        "success": not timed_out,
+        "error": timeout_error,
+        "timeout": timeout,
+    }
 
 
 @click.command()
 @click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
-@click.option("--before", type=float, help="Remove only URLs bookmarked before timestamp")
-@click.option("--after", type=float, help="Remove only URLs bookmarked after timestamp")
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(("exact", "substring", "domain", "regex", "tag")),
-    default="exact",
-    help="Type of pattern matching to use when filtering URLs",
-)
-@click.argument("filter_patterns", nargs=-1)
+@click.option("--delete", is_flag=True, help="Compatibility no-op; archived content is deleted automatically")
+@click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots")
+@snapshot_filter_options(default_filter_type="exact")
 @docstring(remove.__doc__)
 def main(**kwargs):
     """Remove the specified URLs from the archive"""
-    remove(**kwargs)
+    result = remove(**kwargs)
+    if not result["success"]:
+        raise SystemExit(124)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_run.py

@@ -40,14 +40,23 @@ Examples:
 __package__ = "archivebox.cli"
 __command__ = "archivebox run"
 
-import sys
 import asyncio
+import os
+import signal
+import sys
 from collections import defaultdict
 
 import rich_click as click
 from rich import print as rprint
 
 
+RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
+
+
+def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
+    os._exit(128 + int(sig))
+
+
 def process_stdin_records() -> int:
     """
     Process JSONL records from stdin.
@@ -234,11 +243,17 @@ def process_stdin_records() -> int:
                     crawl_id,
                     snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
                     selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
+                    selected_plugins_are_explicit=False,
                 )
     return 0
 
 
-def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int:
+def run_runner(
+    daemon: bool = False,
+    crawl_id: str | None = None,
+    maintenance_only: bool = False,
+    maintenance_batch_size: int | None = None,
+) -> int:
     """
     Run the background runner loop.
 
@@ -250,21 +265,34 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
     from archivebox.config import CONSTANTS
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Machine, Process
-    from archivebox.core.takeover_util import enter_single_runner_gate
-    from archivebox.services.runner import recover_orchestrator_state, run_pending_crawls
+    from archivebox.core.takeover_util import enter_single_runner_gate, standby_until_foreground_runner_needed
+    from archivebox.core.recovery_util import recover_orchestrator_state
+    from archivebox.services.runner import run_pending_crawls
 
     Machine.current()
     current = Process.current()
+    root_command = current.root
+    if daemon and root_command.process_type in (
+        Process.TypeChoices.SERVER,
+        Process.TypeChoices.ADD,
+        Process.TypeChoices.UPDATE,
+    ):
+        # Server-owned daemon runners are persistent supervisor workers, but
+        # foreground add/update commands are allowed to borrow runner/sonic
+        # leadership without taking down Daphne. Waiting here keeps the worker
+        # on the normal runner path while preventing a server restart loop from
+        # immediately stealing the single-runner gate back from the newer CLI.
+        standby_until_foreground_runner_needed(root_command, data_dir=CONSTANTS.DATA_DIR)
     if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
         current.mark_exited()
         return 0
 
-    recover_orchestrator_state(include_chrome=True)
+    recover_orchestrator_state(include_chrome=crawl_id is None, crawl_id=crawl_id)
     if crawl_id:
         from django.utils import timezone
         from archivebox.crawls.models import Crawl
 
-        crawl = Crawl.objects.filter(id=crawl_id, status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED]).first()
+        crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
         now = timezone.now()
         # Only re-lease when the row is unscheduled (retry_at IS NULL) or its
         # existing lease has already expired. A future retry_at means another
@@ -281,16 +309,30 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
     # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
     # orchestrators should shut down immediately and cleanly on the first signal.
     interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
+    if daemon:
+        os.environ[RUNNER_DAEMON_ENV] = "1"
     try:
-        with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
+        with (
+            foreground_shutdown_signals(
+                on_signal=_exit_daemon_runner_on_signal if daemon else None,
+                raise_on_first_signal=not daemon,
+            ),
+            foreground_parent_watchdog(enabled=not daemon),
+        ):
             run_pending_crawls(
                 daemon=daemon,
                 crawl_id=crawl_id,
                 maintenance_only=maintenance_only,
                 interactive_interrupts=interactive_interrupts,
+                **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
             )
         return 0
-    except (KeyboardInterrupt, asyncio.CancelledError):
+    except KeyboardInterrupt:
+        return 0
+    except asyncio.CancelledError as e:
+        if daemon:
+            rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+            return 1
         return 0
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
@@ -307,7 +349,23 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
 @click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
-def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool):
+@click.option(
+    "--maintenance-batch-size",
+    type=int,
+    default=None,
+    hidden=True,
+    help="Limit queued maintenance snapshots claimed per scheduler tick",
+)
+@click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
+def main(
+    daemon: bool,
+    crawl_id: str,
+    snapshot_id: str,
+    binary_id: str,
+    maintenance_only: bool,
+    maintenance_batch_size: int | None,
+    no_stdin: bool,
+):
     """
     Process queued work.
 
@@ -322,8 +380,15 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
 
     if daemon and not snapshot_id and not binary_id and not crawl_id:
         try:
-            with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=False):
-                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
+            os.environ[RUNNER_DAEMON_ENV] = "1"
+            with (
+                foreground_shutdown_signals(
+                    on_signal=_exit_daemon_runner_on_signal,
+                    raise_on_first_signal=False,
+                ),
+                foreground_parent_watchdog(enabled=False),
+            ):
+                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
         except KeyboardInterrupt:
             sys.exit(0)
 
@@ -347,30 +412,49 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
                 sys.exit(1)
 
         if crawl_id:
-            sys.exit(run_runner(daemon=False, crawl_id=crawl_id, maintenance_only=maintenance_only))
+            sys.exit(
+                run_runner(
+                    daemon=False,
+                    crawl_id=crawl_id,
+                    maintenance_only=maintenance_only,
+                    maintenance_batch_size=maintenance_batch_size,
+                ),
+            )
 
         if maintenance_only:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=True))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=True, maintenance_batch_size=maintenance_batch_size))
 
-        if not sys.stdin.isatty():
+        if not no_stdin and not sys.stdin.isatty():
             sys.exit(process_stdin_records())
         else:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
 
 
 def run_snapshot_worker(snapshot_id: str) -> int:
+    from archivebox.config import CONSTANTS
+    from archivebox.core.takeover_util import enter_single_runner_gate
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    from archivebox.machine.models import Process
     from archivebox.core.models import Snapshot
     from archivebox.services.runner import run_due_snapshot
     from django.utils import timezone
 
+    current = Process.current()
+    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
+        current.mark_exited()
+        return 0
+
     snapshot = None
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
-            snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-            if snapshot.retry_at is None:
-                snapshot.update_and_requeue(retry_at=timezone.now())
-            run_due_snapshot(snapshot, lock_seconds=60)
+            for _ in range(10):
+                snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+                if snapshot.retry_at is None:
+                    snapshot.update_and_requeue(retry_at=timezone.now())
+                elif snapshot.retry_at > timezone.now():
+                    break
+                if not run_due_snapshot(snapshot, lock_seconds=60):
+                    break
         return 0
     except KeyboardInterrupt:
         try:
@@ -389,6 +473,10 @@ def run_snapshot_worker(snapshot_id: str) -> int:
 
         traceback.print_exc()
         return 1
+    finally:
+        current.refresh_from_db()
+        if current.status != Process.StatusChoices.EXITED:
+            current.mark_exited()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_search.py

@@ -3,255 +3,7 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox search"
 
-import sys
-from pathlib import Path
-from typing import TYPE_CHECKING
-from collections.abc import Callable
-
-import rich_click as click
-
-from django.db.models import Q, QuerySet
-
-from archivebox.config import DATA_DIR
-from archivebox.config.common import get_config
-from archivebox.misc.logging import stderr
-from archivebox.misc.util import enforce_types, docstring
-
-if TYPE_CHECKING:
-    from archivebox.core.models import Snapshot
-
-# Filter types for URL matching
-LINK_FILTERS: dict[str, Callable[[str], Q]] = {
-    "exact": lambda pattern: Q(url=pattern),
-    "substring": lambda pattern: Q(url__icontains=pattern),
-    "regex": lambda pattern: Q(url__iregex=pattern),
-    "domain": lambda pattern: (
-        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
-    ),
-    "tag": lambda pattern: Q(tags__name=pattern),
-    "timestamp": lambda pattern: Q(timestamp=pattern),
-}
-
-STATUS_CHOICES = ["indexed", "archived", "unarchived"]
-
-
-def _apply_pattern_filters(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    filter_patterns: list[str],
-    filter_type: str,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    filter_builder = LINK_FILTERS.get(filter_type)
-    if filter_builder is None:
-        stderr()
-        stderr(f"[X] Got invalid pattern for --filter-type={filter_type}", color="red")
-        raise SystemExit(2)
-
-    query = Q()
-    for pattern in filter_patterns:
-        query |= filter_builder(pattern)
-    return snapshots.filter(query)
-
-
-def _snapshots_to_json(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from archivebox.config import VERSION
-    from archivebox.misc.util import to_json
-
-    config = get_config()
-    main_index_header = (
-        {
-            "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
-            "schema": "archivebox.index.json",
-            "copyright_info": config.FOOTER_INFO,
-            "meta": {
-                "project": "ArchiveBox",
-                "version": VERSION,
-                "git_sha": VERSION,
-                "website": "https://ArchiveBox.io",
-                "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
-                "source": "https://github.com/ArchiveBox/ArchiveBox",
-                "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
-                "dependencies": {},
-            },
-        }
-        if with_headers
-        else {}
-    )
-
-    snapshot_dicts = [snapshot.to_dict(extended=True) for snapshot in snapshots.iterator(chunk_size=500)]
-    output: dict[str, object] | list[dict[str, object]]
-    if with_headers:
-        output = {
-            **main_index_header,
-            "num_links": len(snapshot_dicts),
-            "updated": datetime.now(tz.utc),
-            "last_run_cmd": sys.argv,
-            "links": snapshot_dicts,
-        }
-    else:
-        output = snapshot_dicts
-
-    return to_json(output, indent=4, sort_keys=True)
-
-
-def _snapshots_to_csv(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    cols: list[str],
-    with_headers: bool,
-) -> str:
-    header = ",".join(cols) if with_headers else ""
-    rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.iterator(chunk_size=500)]
-    return "\n".join((header, *rows))
-
-
-def _snapshots_to_html(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from django.template.loader import render_to_string
-
-    from archivebox.config import VERSION
-    from archivebox.config.version import get_COMMIT_HASH
-
-    config = get_config()
-    template = "static_index.html" if with_headers else "minimal_index.html"
-    snapshot_list = list(snapshots.iterator(chunk_size=500))
-
-    return render_to_string(
-        template,
-        {
-            "version": VERSION,
-            "git_sha": get_COMMIT_HASH() or VERSION,
-            "num_links": str(len(snapshot_list)),
-            "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"),
-            "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"),
-            "links": snapshot_list,
-            "FOOTER_INFO": config.FOOTER_INFO,
-        },
-    )
-
-
-def get_snapshots(
-    snapshots: QuerySet["Snapshot", "Snapshot"] | None = None,
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    after: float | None = None,
-    before: float | None = None,
-    out_dir: Path = DATA_DIR,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    """Filter and return Snapshots matching the given criteria."""
-    from archivebox.core.models import Snapshot
-
-    if snapshots is not None:
-        result = snapshots
-    else:
-        result = Snapshot.objects.all()
-
-    if after is not None:
-        result = result.filter(timestamp__gte=after)
-    if before is not None:
-        result = result.filter(timestamp__lt=before)
-    if filter_patterns:
-        result = _apply_pattern_filters(result, filter_patterns, filter_type)
-
-    # Prefetch crawl relationship to avoid N+1 queries when accessing output_dir
-    result = result.select_related("crawl", "crawl__created_by")
-
-    if not result.exists():
-        stderr("[!] No Snapshots matched your filters:", filter_patterns, f"({filter_type})", color="lightyellow")
-
-    return result
-
-
-@enforce_types
-def search(
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    status: str = "indexed",
-    before: float | None = None,
-    after: float | None = None,
-    sort: str | None = None,
-    json: bool = False,
-    html: bool = False,
-    csv: str | None = None,
-    with_headers: bool = False,
-):
-    """List, filter, and export information about archive entries"""
-
-    if with_headers and not (json or html or csv):
-        stderr("[X] --with-headers requires --json, --html or --csv\n", color="red")
-        raise SystemExit(2)
-
-    # Query DB directly - no filesystem scanning
-    snapshots = get_snapshots(
-        filter_patterns=list(filter_patterns) if filter_patterns else None,
-        filter_type=filter_type,
-        before=before,
-        after=after,
-    )
-
-    # Apply status filter
-    if status == "archived":
-        snapshots = snapshots.filter(downloaded_at__isnull=False)
-    elif status == "unarchived":
-        snapshots = snapshots.filter(downloaded_at__isnull=True)
-    # 'indexed' = all snapshots (no filter)
-
-    if sort:
-        snapshots = snapshots.order_by(sort)
-
-    # Export to requested format
-    if json:
-        output = _snapshots_to_json(snapshots, with_headers=with_headers)
-    elif html:
-        output = _snapshots_to_html(snapshots, with_headers=with_headers)
-    elif csv:
-        output = _snapshots_to_csv(snapshots, cols=csv.split(","), with_headers=with_headers)
-    else:
-        from archivebox.misc.logging_util import printable_folders
-
-        # Convert to dict for printable_folders
-        folders: dict[str, Snapshot | None] = {str(snapshot.output_dir): snapshot for snapshot in snapshots}
-        output = printable_folders(folders, with_headers)
-
-    # Structured exports must be written directly to stdout.
-    # rich.print() reflows long lines to console width, which corrupts JSON/CSV/HTML output.
-    sys.stdout.write(output)
-    if not output.endswith("\n"):
-        sys.stdout.write("\n")
-    return output
-
-
-@click.command()
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(["search", *LINK_FILTERS.keys()]),
-    default="substring",
-    help="Pattern matching type for filtering URLs",
-)
-@click.option("--status", "-s", type=click.Choice(STATUS_CHOICES), default="indexed", help="List snapshots with the given status")
-@click.option("--before", "-b", type=float, help="List snapshots bookmarked before the given UNIX timestamp")
-@click.option("--after", "-a", type=float, help="List snapshots bookmarked after the given UNIX timestamp")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--json", "-J", is_flag=True, help="Print output in JSON format")
-@click.option("--html", "-M", is_flag=True, help="Print output in HTML format (suitable for viewing statically without a server)")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: created_at,url,title")
-@click.option("--with-headers", "-H", is_flag=True, help="Include extra CSV/HTML headers in the output")
-@click.help_option("--help", "-h")
-@click.argument("filter_patterns", nargs=-1)
-@docstring(search.__doc__)
-def main(**kwargs):
-    return search(**kwargs)
+from archivebox.cli.archivebox_list import main
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_server.py

@@ -12,6 +12,7 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print
 
+from archivebox.config import CONSTANTS
 from archivebox.misc.util import docstring, enforce_types
 
 
@@ -162,7 +163,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
     # CSRF_TRUSTED_ORIGINS set, get_base_url() will silently use that as the
     # implicit BASE_URL. Surface what we picked so the user knows where their
     # links / redirects are going — and tell them how to make it explicit.
-    from archivebox.core.host_util import derive_base_url_from_csrf
+    from archivebox.core.routes_util import derive_base_url_from_csrf
 
     csrf_derived = derive_base_url_from_csrf(config)
     if csrf_derived:
@@ -178,7 +179,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
         print()
         return
 
-    # BASE_URL was not set explicitly. The host_util derivation gives one of
+    # BASE_URL was not set explicitly. The routes_util derivation gives one of
     # three results, with very different risk profiles — show a tailored hint
     # so new users coming from the 0.7.x single-domain world know whether the
     # default is fine for them or needs attention.
@@ -203,7 +204,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
             )
         print()
     else:
-        # Loopback / wildcard bind. The host_util default of
+        # Loopback / wildcard bind. The routes_util default of
         # http://archivebox.localhost:PORT works in a browser on the same
         # machine, but anything else (reverse proxy, k8s ingress, LAN client)
         # needs BASE_URL set. (Real hostnames can't reach this branch — the
@@ -230,6 +231,7 @@ def server(
     debug: bool = False,
     daemonize: bool = False,
     nothreading: bool = False,
+    createsuperuser: bool = False,
 ) -> None:
     """Run the ArchiveBox HTTP server"""
     from archivebox.config.common import get_config
@@ -243,6 +245,12 @@ def server(
 
     from django.contrib.auth.models import User
 
+    if createsuperuser:
+        from archivebox.cli.archivebox_manage import manage
+
+        manage(args=["createsuperuser"])
+        print()
+
     if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
         print()
         print(
@@ -259,8 +267,6 @@ def server(
     host, port = _parse_and_validate_bind_spec(bind_spec)
 
     if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1":
-        from archivebox.config import CONSTANTS
-
         log_path = CONSTANTS.LOGS_DIR / "server.log"
         log_path.parent.mkdir(parents=True, exist_ok=True)
         daemon_env = os.environ.copy()
@@ -298,7 +304,7 @@ def server(
         return
 
     os.environ["BIND_ADDR"] = f"{host}:{port}"
-    from archivebox.core.host_util import get_base_url
+    from archivebox.core.routes_util import get_base_url
 
     base_url = get_base_url().rstrip("/")
     admin_url = f"{base_url}/admin/"
@@ -314,6 +320,7 @@ def server(
     from archivebox.core.takeover_util import (
         command_owns_runtime_stack,
         current_command,
+        foreground_runner_owner,
         runtime_stack_owner,
         standby_until_runtime_stack_needed,
     )
@@ -337,21 +344,28 @@ def server(
     runtime_config = get_config()
     _print_server_startup_warnings(runtime_config, host, port)
     bind_url = f"http://{host}:{port}"
-    command = current_command(Process.TypeChoices.SERVER, data_dir=config.DATA_DIR, url=bind_url)
+    command = current_command(Process.TypeChoices.SERVER, data_dir=CONSTANTS.DATA_DIR, url=bind_url)
 
     def still_owns_runtime_stack() -> bool:
         from django.db import connections
 
         try:
-            return command_owns_runtime_stack(command, data_dir=config.DATA_DIR)
+            return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
         finally:
             connections.close_all()
 
+    shutdown_state = None
     try:
-        with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"):
+        with (
+            foreground_shutdown_signals() as shutdown_state,
+            foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"),
+        ):
             while True:
-                standby_result = standby_until_runtime_stack_needed(command, data_dir=config.DATA_DIR)
-                older_owner = runtime_stack_owner(data_dir=config.DATA_DIR, exclude_id=command.id)
+                standby_result = standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id) or foreground_runner_owner(
+                    data_dir=CONSTANTS.DATA_DIR,
+                    exclude_id=command.id,
+                )
                 takeover_components = active_supervisord_runtime_components(config=config)
                 if older_owner and takeover_components:
                     print(
@@ -386,7 +400,8 @@ def server(
     except KeyboardInterrupt:
         pass
     finally:
-        command.mark_exited()
+        if not shutdown_state or not shutdown_state.signal_name:
+            command.mark_exited()
     print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")
 
 
@@ -396,6 +411,7 @@ def server(
 @click.option("--debug", is_flag=True, help="Enable DEBUG=True mode with more verbose errors")
 @click.option("--nothreading", is_flag=True, help="Force runserver to run in single-threaded mode")
 @click.option("--daemonize", is_flag=True, help="Run the server in the background as a daemon")
+@click.option("--createsuperuser", is_flag=True, help="Run archivebox manage createsuperuser before starting the server")
 @docstring(server.__doc__)
 def main(**kwargs):
     server(**kwargs)

archivebox/cli/archivebox_shell.py

@@ -12,9 +12,10 @@ from archivebox.misc.util import docstring
 def shell(args: Iterable[str] = ()) -> None:
     """Enter an interactive ArchiveBox Django shell"""
 
-    from django.core.management import call_command
+    from django.core.management import call_command, get_commands
 
-    call_command("shell_plus", *args)
+    shell_command = "shell_plus" if "shell_plus" in get_commands() else "shell"
+    call_command(shell_command, *args)
 
 
 @click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))

archivebox/cli/archivebox_snapshot.py

@@ -35,9 +35,10 @@ from collections.abc import Iterable
 
 import rich_click as click
 from rich import print as rprint
-from django.db.models import Case, IntegerField, Q, QuerySet, When
+from django.db.models import QuerySet
 
-from archivebox.cli.cli_util import apply_filters
+SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp")
+SNAPSHOT_LIST_CHUNK_SIZE = 100
 
 
 # =============================================================================
@@ -66,7 +67,7 @@ def create_snapshots(
         TYPE_SNAPSHOT,
         TYPE_CRAWL,
     )
-    from archivebox.misc.util import validate_url_length
+    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot
     from archivebox.crawls.models import Crawl
@@ -111,7 +112,7 @@ def create_snapshots(
                 # Create snapshots for each URL in the crawl
                 for url in crawl.get_urls_list():
                     try:
-                        validate_url_length(url)
+                        validate_url(url)
                     except ValueError as err:
                         rprint(f"[red]Error creating snapshot: {err}[/red]", file=sys.stderr)
                         continue
@@ -134,7 +135,7 @@ def create_snapshots(
             elif record_type == TYPE_SNAPSHOT or record.get("url"):
                 # Input is a Snapshot or plain URL
                 if record.get("url"):
-                    validate_url_length(str(record["url"]))
+                    validate_url(str(record["url"]))
                 if tag and not record.get("tags"):
                     record["tags"] = tag
                 if status:
@@ -178,94 +179,63 @@ def create_snapshots(
 # =============================================================================
 
 
+def snapshot_filter_options(*, default_filter_type: str):
+    def decorate(func):
+        for decorator in reversed(
+            (
+                click.option("--status", "-s", help="Filter by status (queued, started, sealed)"),
+                click.option("--url__icontains", help="Filter by URL contains"),
+                click.option("--url__istartswith", help="Filter by URL starts with"),
+                click.option("--tag", "-t", help="Filter by tag name"),
+                click.option("--crawl-id", help="Filter by crawl ID"),
+                click.option("--limit", "-n", type=int, help="Limit number of results"),
+                click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at"),
+                click.option("--search", help="Search mode to use for positional query"),
+                click.option("--before", type=float, help="Only snapshots bookmarked before timestamp"),
+                click.option("--after", type=float, help="Only snapshots bookmarked after timestamp"),
+                click.option(
+                    "--filter-type",
+                    "-f",
+                    type=click.Choice(SNAPSHOT_FILTER_TYPE_CHOICES),
+                    default=default_filter_type,
+                    help="Type of pattern matching to use for positional filters",
+                ),
+                click.argument("filter_patterns", nargs=-1),
+            ),
+        ):
+            func = decorator(func)
+        return func
+
+    return decorate
+
+
+def snapshot_output_options(func):
+    for decorator in reversed(
+        (
+            click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title"),
+            click.option("--json", "as_json", is_flag=True, help="Print output as a JSON array"),
+            click.option("--html", "as_html", is_flag=True, help="Print output as HTML"),
+            click.option("--with-headers", is_flag=True, help="Include column headers in structured output"),
+        ),
+    ):
+        func = decorator(func)
+    return func
+
+
 def build_snapshot_queryset(
-    *,
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    sort: str | None = None,
-    search: str | None = None,
-    query: str | None = None,
-    limit: int | None = None,
+    **kwargs,
 ) -> QuerySet:
     from archivebox.core.models import Snapshot
-    from archivebox.search import (
-        get_default_search_mode,
-        get_search_mode,
-        prioritize_metadata_matches,
-        query_search_index,
-    )
 
-    queryset = Snapshot.objects.order_by("-created_at")
-    queryset = apply_filters(
-        queryset,
-        {
-            "status": status,
-            "url__icontains": url__icontains,
-            "url__istartswith": url__istartswith,
-            "crawl_id": crawl_id,
-        },
-    )
-
-    if tag:
-        queryset = queryset.filter(tags__name__iexact=tag)
-
-    query = (query or "").strip()
-    if query:
-        metadata_qs = queryset.filter(
-            Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query),
-        )
-        requested_search_mode = (search or "").strip().lower()
-        if requested_search_mode == "content":
-            requested_search_mode = "contents"
-        search_mode = get_default_search_mode() if not requested_search_mode else get_search_mode(requested_search_mode)
-
-        if search_mode == "meta":
-            queryset = metadata_qs
-        elif limit and len(list(metadata_qs.values_list("pk", flat=True).distinct()[:limit])) >= limit:
-            queryset = metadata_qs
-        else:
-            try:
-                deep_qsearch = None
-                if search_mode == "deep":
-                    qsearch = query_search_index(query, search_mode="contents", max_results=limit)
-                    deep_qsearch = query_search_index(query, search_mode="deep", max_results=limit)
-                else:
-                    qsearch = query_search_index(query, search_mode=search_mode, max_results=limit)
-                queryset = prioritize_metadata_matches(
-                    queryset,
-                    metadata_qs,
-                    qsearch,
-                    deep_queryset=deep_qsearch,
-                    ordering=("-created_at",) if not sort else None,
-                )
-            except Exception as err:
-                rprint(
-                    f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
-                    file=sys.stderr,
-                )
-                queryset = metadata_qs
-
-    if sort:
-        queryset = queryset.order_by(sort)
-
-    return queryset
+    return Snapshot.objects.order_by("-created_at").search(**kwargs)
 
 
 def list_snapshots(
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
     csv: str | None = None,
+    as_json: bool = False,
+    as_html: bool = False,
     with_headers: bool = False,
-    search: str | None = None,
-    query: str | None = None,
+    **kwargs,
 ) -> int:
     """
     List Snapshots as JSONL with optional filters.
@@ -274,96 +244,71 @@ def list_snapshots(
         0: Success (even if no results)
     """
     from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import Snapshot
 
-    if with_headers and not csv:
-        rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr)
+    output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
+    if output_formats > 1:
+        rprint("[red]Choose only one output format: --csv, --json, or --html[/red]", file=sys.stderr)
+        return 2
+    if with_headers and not output_formats:
+        rprint("[red]--with-headers requires --csv, --json, or --html[/red]", file=sys.stderr)
         return 2
 
-    is_tty = sys.stdout.isatty() and not csv
+    is_tty = sys.stdout.isatty() and not output_formats
 
-    queryset = build_snapshot_queryset(
-        status=status,
-        url__icontains=url__icontains,
-        url__istartswith=url__istartswith,
-        tag=tag,
-        crawl_id=crawl_id,
-        sort=sort,
-        search=search,
-        query=query,
-        limit=limit,
-    )
-
-    if not is_tty:
-        if limit:
-            limited_ids = list(queryset.values_list("id", flat=True)[:limit])
-            preserved_order = Case(
-                *(When(id=snapshot_id, then=position) for position, snapshot_id in enumerate(limited_ids)),
-                output_field=IntegerField(),
-            )
-            queryset = Snapshot.objects.filter(id__in=limited_ids).order_by(preserved_order)
-        queryset = queryset.prefetch_related("tags")
-    elif limit:
-        queryset = queryset[:limit]
+    try:
+        queryset = build_snapshot_queryset(**kwargs)
+    except ValueError as err:
+        rprint(f"[red]{err}[/red]", file=sys.stderr)
+        return 2
 
     count = 0
+    if as_json:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_json(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    if as_html:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_html(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
     if csv:
         cols = [col.strip() for col in csv.split(",") if col.strip()]
         if not cols:
             rprint("[red]No CSV columns provided[/red]", file=sys.stderr)
             return 2
-        rows: list[str] = []
         if with_headers:
-            rows.append(",".join(cols))
-        simple_cols = {
-            "id",
-            "crawl_id",
-            "url",
-            "title",
-            "timestamp",
-            "depth",
-            "status",
-            "fs_version",
-            "bookmarked_at",
-            "created_at",
-            "modified_at",
-            "retry_at",
-            "downloaded_at",
-        }
-        from archivebox.misc.util import to_json
-
-        for snapshot in queryset.iterator(chunk_size=500):
-            if set(cols).issubset(simple_cols):
-                rows.append(
-                    ",".join(
-                        to_json(
-                            value.isoformat() if hasattr((value := getattr(snapshot, col, "")), "isoformat") else value,
-                            indent=None,
-                        )
-                        for col in cols
-                    ),
-                )
-            else:
-                rows.append(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write(",".join(cols))
+            sys.stdout.write("\n")
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+            sys.stdout.write(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write("\n")
             count += 1
-        output = "\n".join(rows)
-        if output:
-            sys.stdout.write(output)
-            if not output.endswith("\n"):
-                sys.stdout.write("\n")
         rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
         return 0
 
-    for snapshot in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(snapshot.status, "dim")
-            rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
-        else:
+    if not is_tty:
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
             write_record(snapshot.to_json())
+            count += 1
+        rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    for snapshot in queryset.iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+        status_color = {
+            "queued": "yellow",
+            "started": "blue",
+            "sealed": "green",
+        }.get(snapshot.status, "dim")
+        rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
         count += 1
 
     rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
@@ -522,46 +467,11 @@ def create_cmd(urls: tuple, tag: str, status: str, depth: int):
 
 
 @main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def list_cmd(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-):
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def list_cmd(**kwargs):
     """List Snapshots as JSONL."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))
 
 
 @main.command("update")

archivebox/cli/archivebox_snapshot_compat.py

@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox snapshot"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_snapshot import create_snapshots
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--depth", "-d", type=int, default=0, help="Crawl depth (default: 0)")
-@click.argument("urls", nargs=-1)
-def main(tag: str, status: str, depth: int, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox snapshot URL...` entrypoint."""
-    sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth))
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_status.py

@@ -8,7 +8,7 @@ import rich_click as click
 from rich import print
 
 from archivebox.misc.util import enforce_types, docstring
-from archivebox.config import DATA_DIR, CONSTANTS
+from archivebox.config import CONSTANTS
 from archivebox.config.common import get_config
 from archivebox.misc.system import get_dir_size
 from archivebox.misc.logging_util import printable_filesize
@@ -18,7 +18,7 @@ MAX_STATUS_FS_DIR_SCAN = 5000
 
 
 @enforce_types
-def status(out_dir: Path = DATA_DIR) -> None:
+def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
     """Print out some info and statistics about the archive collection"""
 
     from django.contrib.auth import get_user_model
@@ -38,7 +38,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
 
     snapshots_qs = Snapshot.objects.all()
     num_sql_links = snapshots_qs.count()
-    archive_dir = config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
     legacy_snapshot_dirs = []
     if archive_dir.exists():
         legacy_snapshot_dirs = [
@@ -48,7 +48,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
     print(f"    > JSON Link Details: {len(legacy_snapshot_dirs)} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)")
     print()
     print("[green]\\[*] Scanning archive data directories...[/green]")
-    users_dir = config.USERS_DIR
+    users_dir = CONSTANTS.USERS_DIR
     scan_roots = [root for root in (archive_dir, users_dir) if root.exists()]
     scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(archive_dir)
     print(f"[yellow]   {scan_roots_display}[/yellow]")
@@ -61,7 +61,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
             num_dirs += root_dirs
             num_files += root_files
     else:
-        num_bytes = ArchiveResult.objects.aggregate(total=Coalesce(Sum("output_size"), 0))["total"] or 0
+        num_bytes = snapshots_qs.aggregate(total=Coalesce(Sum("output_size"), 0))["total"] or 0
         num_dirs = 0
         num_files = ArchiveResult.objects.exclude(output_files__in=["", "{}"]).count()
     size = printable_filesize(num_bytes)
@@ -105,7 +105,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
 
     if num_indexed:
         print("    [violet]Hint:[/violet] You can list snapshots by status like so:")
-        print("        [green]archivebox list --status=<status>  (e.g. archived, queued, etc.)[/green]")
+        print("        [green]archivebox list --status=<status>  (e.g. sealed, queued, etc.)[/green]")
 
     if orphaned_dirs:
         print("    [violet]Hint:[/violet] To automatically import orphaned data directories into the main index, run:")

archivebox/cli/archivebox_update.py

@@ -15,6 +15,7 @@ from pathlib import Path
 import rich_click as click
 
 from archivebox.misc.util import enforce_types, docstring
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
@@ -32,66 +33,36 @@ def _get_snapshot_crawl(snapshot: Snapshot) -> Crawl | None:
 
 
 def _get_search_indexing_plugins() -> list[str]:
-    from abx_dl.models import discover_plugins
-    from archivebox.hooks import get_search_backends
+    from archivebox.config.common import get_config
+    from archivebox.plugins.hooks import discover_hooks
+    from archivebox.plugins.discovery import get_search_backends
 
     available_backends = set(get_search_backends())
-    plugins = discover_plugins()
     return sorted(
         plugin_name
-        for plugin_name, plugin in plugins.items()
-        if plugin_name.startswith("search_backend_")
-        and plugin_name.removeprefix("search_backend_") in available_backends
-        and any("Snapshot" in hook.name and "index" in hook.name.lower() for hook in plugin.hooks)
+        for plugin_name in {
+            hook.parent.name
+            for hook in discover_hooks("Snapshot", config=get_config())
+            if hook.parent.name.startswith("search_backend_") and "index" in hook.name.lower()
+        }
+        if plugin_name.startswith("search_backend_") and plugin_name.removeprefix("search_backend_") in available_backends
     )
 
 
 def _build_filtered_snapshots_queryset(
-    *,
-    filter_patterns: Iterable[str],
-    filter_type: str,
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
-    search: str | None = None,
-    before: float | None = None,
-    after: float | None = None,
-    resume: str | None = None,
+    **kwargs,
 ):
-    from datetime import datetime
+    from archivebox.core.models import Snapshot
     from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
-    filter_patterns = tuple(filter_patterns)
-    snapshots = build_snapshot_queryset(
-        status=status,
-        url__icontains=url__icontains,
-        url__istartswith=url__istartswith,
-        tag=tag,
-        crawl_id=crawl_id,
-        sort=sort,
-        search=search,
-        query=" ".join(filter_patterns) if search else None,
-    )
-
-    if filter_patterns and not search:
-        snapshots = snapshots.filter_by_patterns(list(filter_patterns), filter_type)
-
-    if before:
-        snapshots = snapshots.filter(bookmarked_at__lt=datetime.fromtimestamp(before))
-    if after:
-        snapshots = snapshots.filter(bookmarked_at__gt=datetime.fromtimestamp(after))
-    if resume:
-        snapshots = snapshots.filter(timestamp__lte=resume)
-    if not sort:
-        snapshots = snapshots.order_by("-timestamp")
+    limit = kwargs.pop("limit", None)
+    snapshots = build_snapshot_queryset(**kwargs)
+    if kwargs.get("resume"):
+        snapshots = snapshots.filter(timestamp__lte=kwargs["resume"])
     snapshots = snapshots.select_related("crawl")
-    if limit:
-        limited_ids = list(snapshots.values_list("id", flat=True)[:limit])
-        snapshots = snapshots.model.objects.filter(id__in=limited_ids).select_related("crawl")
+    if limit is not None and limit > 0:
+        snapshot_ids = list(snapshots.values_list("id", flat=True)[:limit])
+        snapshots = Snapshot.objects.filter(id__in=snapshot_ids).select_related("crawl")
 
     return snapshots
 
@@ -105,9 +76,17 @@ def reindex_snapshots(
     wait_for_turn=None,
 ) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
+    from archivebox.core.models import ArchiveResult
+    from abx_dl.models import discover_plugins
 
-    stats: dict[str, Any] = {"processed": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
+    stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
     records: list[dict[str, str]] = []
+    plugins_by_name = discover_plugins(runtime="archivebox")
+    required_hooks_by_plugin = {
+        plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
+        for plugin_name in search_plugins
+        if plugin_name in plugins_by_name
+    }
 
     total = snapshots.count()
     print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}")
@@ -118,6 +97,31 @@ def reindex_snapshots(
         if wait_for_turn:
             wait_for_turn()
         batch_records = list(records)
+        snapshot_ids = {record["snapshot_id"] for record in batch_records}
+        plugin_names = {record["plugin"] for record in batch_records}
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=snapshot_ids,
+                plugin__in=plugin_names,
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        records_to_queue = []
+        for record in batch_records:
+            snapshot_id = record["snapshot_id"]
+            plugin_name = record["plugin"]
+            required_hooks = required_hooks_by_plugin.get(plugin_name, frozenset())
+            if required_hooks and all((snapshot_id, plugin_name, hook_name) in queued_rows for hook_name in required_hooks):
+                stats["skipped_queued"] += 1
+                continue
+            records_to_queue.append(record)
+        if not records_to_queue:
+            print(
+                f"    [{stats['processed']}/{total}] Already queued {len(batch_records)} index jobs",
+            )
+            records.clear()
+            return
         # `archivebox update --index-only` intentionally breaks the usual
         # "runner discovers work" rule by inserting synthetic queued
         # ArchiveResult rows for search backends. run_plugins() keeps this as
@@ -127,15 +131,17 @@ def reindex_snapshots(
         # finish.
         exit_code = run_plugins(
             args=(),
-            records=batch_records,
+            records=records_to_queue,
             wait=False,
             emit_results=False,
             show_progress=False,
+            preserve_queued=True,
         )
         if exit_code != 0:
             raise SystemExit(exit_code)
+        stats["queued"] += len(records_to_queue)
         print(
-            f"    [{stats['processed']}/{total}] Queued {len(batch_records)} index jobs for orchestrator",
+            f"    [{stats['processed']}/{total}] Queued {len(records_to_queue)} index jobs for orchestrator",
         )
         records.clear()
 
@@ -156,7 +162,7 @@ def reindex_snapshots(
                         "plugin": plugin_name,
                     },
                 )
-                stats["queued"] += 1
+                stats["requested"] += 1
             if len(records) >= batch_size:
                 run_batch()
         except KeyboardInterrupt as err:
@@ -219,27 +225,42 @@ def update(
     from archivebox.machine.models import Process
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals, raise_if_shutdown_requested
     from archivebox.core.takeover_util import (
-        command_owns_runtime_stack,
+        command_owns_foreground_runner,
         current_command,
         ensure_daemon_stack,
-        standby_until_runtime_stack_needed,
+        foreground_runner_owner,
+        standby_until_foreground_runner_needed,
     )
-    from archivebox.workers.supervisord_util import run_runner_worker, stop_existing_supervisord_process, stop_own_supervisord_process
+    from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
 
     command = current_command(Process.TypeChoices.UPDATE, data_dir=CONSTANTS.DATA_DIR)
 
+    def still_owns_foreground_runner() -> bool:
+        from django.db import connections
+
+        try:
+            return command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR)
+        finally:
+            connections.close_all()
+
     def wait_for_turn() -> None:
         raise_if_shutdown_requested()
-        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
         raise_if_shutdown_requested()
 
-    def run_scoped_runner(*args: str) -> None:
+    def run_scoped_runner(*args: str, ensure_daemon_reason: str | None = None) -> None:
         while True:
             wait_for_turn()
-            exit_code = run_runner_worker(list(args), name=f"worker_runner_update_{os.getpid()}")
+            if ensure_daemon_reason:
+                ensure_daemon_stack(reason=ensure_daemon_reason)
+            exit_code = run_runner_worker(
+                list(args),
+                name=f"worker_runner_update_{os.getpid()}",
+                keep_running=still_owns_foreground_runner,
+            )
             if exit_code == 0:
                 return
-            if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+            if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
                 continue
             raise SystemExit(exit_code)
 
@@ -263,8 +284,6 @@ def update(
 
     try:
         wait_for_turn()
-        if stop_daemon_stack:
-            stop_existing_supervisord_process()
 
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             while True:
@@ -272,6 +291,9 @@ def update(
                 do_index = index_only or not migrate_only
                 do_run_until_idle = do_migrate or do_index
                 ran_post_migrate_runner = False
+                full_update_empty = False
+                maintenance_work_queued = False
+                runner_work_queued = False
 
                 if do_migrate:
                     if (
@@ -308,6 +330,8 @@ def update(
                         )
                         print_stats(stats)
                         touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                        maintenance_work_queued = stats.get("queued", 0) > 0
+                        runner_work_queued = runner_work_queued or maintenance_work_queued
                     else:
                         stats_combined = {"phase1": {}, "phase2": {}}
 
@@ -324,6 +348,17 @@ def update(
                             wait_for_turn=wait_for_turn,
                         )
                         print_combined_stats(stats_combined)
+                        full_update_empty = (
+                            stats_combined["phase1"].get("processed", 0) == 0 and stats_combined["phase2"].get("snapshots", 0) == 0
+                        )
+                        maintenance_work_queued = any(
+                            (
+                                stats_combined["phase1"].get("queued", 0),
+                                stats_combined["phase2"].get("queued", 0),
+                                stats_combined["phase2"].get("crawls_sealed", 0),
+                            ),
+                        )
+                        runner_work_queued = runner_work_queued or maintenance_work_queued
 
                     if do_run_until_idle:
                         # Filesystem migration is maintenance on existing
@@ -335,46 +370,108 @@ def update(
                         # rows first, runs the targeted plugins, and may leave
                         # the fs_version maintenance tick hidden behind that
                         # plugin work until another update pass.
-                        print("[*] Phase 3: Running filesystem maintenance until idle...")
-                        if is_filtered_update:
+                        if full_update_empty:
+                            print("[*] No snapshots or legacy archive directories found; skipping filesystem maintenance runner.")
+                        elif not maintenance_work_queued:
+                            print("[*] No filesystem maintenance work queued; skipping filesystem maintenance runner.")
+                        else:
+                            print("[*] Phase 3: Running filesystem maintenance until idle...")
+                        if full_update_empty:
+                            pass
+                        elif not maintenance_work_queued:
+                            pass
+                        elif is_filtered_update:
                             if not touched_snapshot_ids:
                                 print("[*] No matching snapshots queued work for the runner.")
                             for snapshot_id in sorted(touched_snapshot_ids):
                                 run_scoped_runner("--snapshot-id", snapshot_id)
                         else:
-                            run_scoped_runner("--maintenance-only")
+                            run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size))
                         ran_post_migrate_runner = True
 
                 if do_index:
-                    ensure_daemon_stack(reason="search indexing")
-                    search_plugins = _get_search_indexing_plugins()
-                    if not search_plugins:
-                        print("[*] No search indexing plugins are available, nothing to backfill.")
+                    if full_update_empty:
+                        print("[*] No snapshots found; skipping search indexing backfill.")
                     else:
-                        snapshots = _build_filtered_snapshots_queryset(
-                            filter_patterns=filter_patterns,
-                            filter_type=filter_type,
-                            status=status,
-                            url__icontains=url__icontains,
-                            url__istartswith=url__istartswith,
-                            tag=tag,
-                            crawl_id=crawl_id,
-                            limit=limit,
-                            sort=sort,
-                            search=search,
-                            before=before,
-                            after=after,
-                            resume=resume,
-                        )
-                        stats = reindex_snapshots(
-                            snapshots,
-                            search_plugins=search_plugins,
-                            batch_size=batch_size,
-                            collect_ids=is_filtered_update,
-                            wait_for_turn=wait_for_turn,
-                        )
-                        print_index_stats(stats)
-                        touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                        ensure_daemon_stack(reason="search indexing")
+                        search_plugins = _get_search_indexing_plugins()
+                        if not search_plugins:
+                            print("[*] No search indexing plugins are available, nothing to backfill.")
+                        else:
+                            snapshots = _build_filtered_snapshots_queryset(
+                                filter_patterns=filter_patterns,
+                                filter_type=filter_type,
+                                status=status,
+                                url__icontains=url__icontains,
+                                url__istartswith=url__istartswith,
+                                tag=tag,
+                                crawl_id=crawl_id,
+                                limit=limit,
+                                sort=sort,
+                                search=search,
+                                before=before,
+                                after=after,
+                                resume=resume,
+                            )
+                            from django.db.models import Exists, OuterRef, Q
+                            from django.utils import timezone
+                            from archivebox.core.models import ArchiveResult, Snapshot
+
+                            scoped_snapshot_ids = snapshots.order_by().values("id") if is_filtered_update else None
+                            queued_index_results = ArchiveResult.objects.filter(
+                                status=ArchiveResult.StatusChoices.QUEUED,
+                                plugin__in=search_plugins,
+                            )
+                            if scoped_snapshot_ids is not None:
+                                queued_index_results = queued_index_results.filter(snapshot_id__in=scoped_snapshot_ids)
+
+                            if queued_index_results.exists():
+                                runner_work_queued = True
+                                now = timezone.now()
+                                queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
+                                snapshots_to_wake = (
+                                    Snapshot.objects.filter(
+                                        status__in=(Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED),
+                                    )
+                                    .annotate(
+                                        has_queued_index_result=Exists(queued_result_for_snapshot),
+                                    )
+                                    .filter(
+                                        has_queued_index_result=True,
+                                    )
+                                    .filter(
+                                        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+                                    )
+                                )
+                                if scoped_snapshot_ids is not None:
+                                    snapshots_to_wake = snapshots_to_wake.filter(id__in=scoped_snapshot_ids)
+                                woken_count = snapshots_to_wake.update(
+                                    retry_at=now,
+                                    modified_at=now,
+                                )
+                                print(
+                                    "[*] Existing queued search index jobs found; "
+                                    f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.",
+                                )
+                            else:
+                                collect_index_ids = (
+                                    is_filtered_update
+                                    or foreground_runner_owner(
+                                        data_dir=CONSTANTS.DATA_DIR,
+                                        exclude_id=command.id,
+                                    )
+                                    is not None
+                                )
+                                stats = reindex_snapshots(
+                                    snapshots,
+                                    search_plugins=search_plugins,
+                                    batch_size=batch_size,
+                                    collect_ids=collect_index_ids,
+                                    wait_for_turn=wait_for_turn,
+                                )
+                                print_index_stats(stats)
+                                touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                                runner_work_queued = runner_work_queued or stats["queued"] > 0
 
                 if do_run_until_idle and (do_index or not ran_post_migrate_runner):
                     # Search/index backfill intentionally queues targeted
@@ -384,14 +481,26 @@ def update(
                     # For a normal unfiltered `archivebox update`, keep the
                     # historical final pass broad enough to resume genuinely
                     # queued/interrupted crawl work after maintenance is done.
-                    print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
-                    if is_filtered_update:
+                    if full_update_empty:
+                        print("[*] No snapshots found; skipping queued/interrupted crawl runner.")
+                    elif not runner_work_queued:
+                        print("[*] No queued/interrupted crawl work found; skipping queued/interrupted crawl runner.")
+                    else:
+                        print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
+                    if full_update_empty:
+                        pass
+                    elif not runner_work_queued:
+                        pass
+                    elif touched_snapshot_ids and is_filtered_update:
                         if not touched_snapshot_ids:
                             print("[*] No matching snapshots queued work for the runner.")
                         for snapshot_id in sorted(touched_snapshot_ids):
                             run_scoped_runner("--snapshot-id", snapshot_id)
                     else:
-                        run_scoped_runner(*(["--maintenance-only"] if index_only or migrate_only else []))
+                        run_scoped_runner(
+                            *(["--maintenance-only", "--maintenance-batch-size", str(batch_size)] if index_only or migrate_only else []),
+                            ensure_daemon_reason="search indexing" if do_index else None,
+                        )
 
                 if not continuous:
                     break
@@ -401,7 +510,7 @@ def update(
                 resume = None
     except (KeyboardInterrupt, asyncio.CancelledError) as err:
         exit_code = 130
-        exact_resume = getattr(err, "archivebox_resume", None)
+        exact_resume = err.__dict__.get("archivebox_resume")
         resume_cmd = ["archivebox", "update"]
         if migrate_only:
             resume_cmd.append("--migrate-only")
@@ -464,7 +573,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
     1:1 mapping between DB and filesystem.
     """
     from archivebox.core.models import Snapshot
-    from archivebox.config.common import get_config
+    from archivebox.config import CONSTANTS
     from archivebox.crawls.models import Crawl
     from django.utils import timezone
 
@@ -473,8 +582,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
     crawl_url_sets: dict[str, set[str]] = {}
     dirty_crawl_ids: set[str] = set()
 
-    runtime_config = get_config()
-    archive_dir = runtime_config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
     if not archive_dir.exists():
         return stats
 
@@ -629,6 +737,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     """
     from archivebox.core.models import Snapshot
     from archivebox.crawls.models import Crawl
+    from django.db.models import Q
     from django.utils import timezone
 
     stats = {
@@ -638,7 +747,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
         "updated_db": 0,
         "queued": 0,
         "sealed": 0,
-        "crawls_queued": 0,
+        "crawls_sealed": 0,
     }
     current_fs_version = Snapshot._fs_current_version()
 
@@ -646,6 +755,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     if resume:
         queryset = queryset.filter(timestamp__lte=resume)
     total = queryset.count()
+    stats["snapshots"] = total
     print(f"[*] Processing {total} snapshots from database (most recent first)...")
 
     def update_in_batches(rows, *, label: str, **updates) -> int:
@@ -694,7 +804,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     stats["sealed"] += updated_rows
     stats["updated_db"] += updated_rows
 
-    fs_version_rows = queryset.exclude(fs_version=current_fs_version)
+    fs_version_rows = queryset.exclude(fs_version=current_fs_version).filter(Q(retry_at__isnull=True) | Q(retry_at__gt=now))
     stale_batch = []
 
     def queue_stale_fs_batch() -> None:
@@ -744,23 +854,23 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     queue_stale_fs_batch()
 
     now = timezone.now()
-    stats["crawls_queued"] = (
+    # Crawls with no open child snapshots are already finished. Seal them here
+    # instead of waking the foreground runner; otherwise migration/update can
+    # accidentally re-enter full crawl execution for historical rows.
+    stats["crawls_sealed"] = (
         Crawl.objects.filter(
-            status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED],
+            status__in=Crawl.RUNNABLE_STATES,
         )
         .exclude(
-            snapshot_set__status__in=[
-                Snapshot.StatusChoices.QUEUED,
-                Snapshot.StatusChoices.STARTED,
-                Snapshot.StatusChoices.PAUSED,
-            ],
+            snapshot_set__status__in=Snapshot.OPEN_STATES,
         )
         .update(
-            retry_at=now,
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
             modified_at=now,
         )
     )
-    stats["updated_db"] += stats["crawls_queued"]
+    stats["updated_db"] += stats["crawls_sealed"]
     return stats
 
 
@@ -898,7 +1008,7 @@ Phase 2 (Process DB):
   Updated JSON:     {s2.get("updated_json", 0)}
   Updated DB rows:  {s2.get("updated_db", 0)}
   Sealed snapshots: {s2.get("sealed", 0)}
-  Queued crawls:    {s2.get("crawls_queued", 0)}
+  Sealed crawls:    {s2.get("crawls_sealed", 0)}
 """)
 
 
@@ -908,28 +1018,19 @@ def print_index_stats(stats: dict[str, Any]) -> None:
     print(f"""
 [green]Search Reindex Complete[/green]
   Scanned rows:      {stats["processed"]}
+  Requested jobs:    {stats.get("requested", stats["queued"])}
   Queued index jobs: {stats["queued"]}
+  Already queued:    {stats.get("skipped_queued", 0)}
 """)
 
 
 @click.command()
 @click.option("--resume", type=str, help="Resume from timestamp")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of snapshots to update")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for positional query")
-@click.option("--before", type=float, help="Only snapshots before timestamp")
-@click.option("--after", type=float, help="Only snapshots after timestamp")
-@click.option("--filter-type", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact")
 @click.option("--batch-size", type=int, default=500, help="Commit every N records")
 @click.option("--continuous", is_flag=True, help="Run continuously as background worker")
 @click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content")
 @click.option("--migrate-only", is_flag=True, help="Only migrate filesystem and update database/index state")
-@click.argument("filter_patterns", nargs=-1)
+@snapshot_filter_options(default_filter_type="exact")
 @docstring(update.__doc__)
 def main(**kwargs):
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
@@ -937,6 +1038,8 @@ def main(**kwargs):
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             update(**kwargs)
+    except ValueError as err:
+        raise click.BadParameter(str(err), param_hint="--status") from err
     except KeyboardInterrupt:
         raise SystemExit(130) from None
 

archivebox/cli/archivebox_version.py

@@ -29,10 +29,8 @@ def _format_binary_abspath(
 
     candidate_bases: tuple[tuple[Path, str], ...] = (
         (pwd, "./"),
-        (lib_dir, "LIB_DIR/"),
-        (Path(os.environ.get("LIB_DIR", "")), "LIB_DIR/") if os.environ.get("LIB_DIR") else (Path(), ""),
+        (lib_dir, "ABXPKG_LIB_DIR/"),
         (personas_dir, "PERSONAS_DIR/"),
-        (Path(os.environ.get("PERSONAS_DIR", "")), "PERSONAS_DIR/") if os.environ.get("PERSONAS_DIR") else (Path(), ""),
         (home, "~/"),
     )
 
@@ -48,8 +46,8 @@ def _format_binary_abspath(
             relative_str = relative.as_posix()
             if prefix == "./":
                 return "." if not relative_str else f"./{relative_str}"
-            if prefix == "LIB_DIR/":
-                return "LIB_DIR" if not relative_str else f"LIB_DIR/{relative_str}"
+            if prefix == "ABXPKG_LIB_DIR/":
+                return "ABXPKG_LIB_DIR" if not relative_str else f"ABXPKG_LIB_DIR/{relative_str}"
             if prefix == "PERSONAS_DIR/":
                 return "PERSONAS_DIR" if not relative_str else f"PERSONAS_DIR/{relative_str}"
             return "~" if not relative_str else f"~/{relative_str}"
@@ -60,10 +58,10 @@ def _format_binary_abspath(
 def _render_binary_abspath(abspath: str):
     from rich.text import Text
 
-    if abspath.startswith("LIB_DIR/"):
-        return Text.assemble(("LIB_DIR", "bright_blue"), (abspath.removeprefix("LIB_DIR"), "green"))
-    if abspath == "LIB_DIR":
-        return Text("LIB_DIR", style="bright_blue")
+    if abspath.startswith("ABXPKG_LIB_DIR/"):
+        return Text.assemble(("ABXPKG_LIB_DIR", "bright_blue"), (abspath.removeprefix("ABXPKG_LIB_DIR"), "green"))
+    if abspath == "ABXPKG_LIB_DIR":
+        return Text("ABXPKG_LIB_DIR", style="bright_blue")
     if abspath.startswith("PERSONAS_DIR/"):
         return Text.assemble(("PERSONAS_DIR", "medium_purple"), (abspath.removeprefix("PERSONAS_DIR"), "green"))
     if abspath == "PERSONAS_DIR":
@@ -79,6 +77,79 @@ def _render_binary_abspath(abspath: str):
     return Text(abspath, style="green")
 
 
+def _build_binary_table(rows: list[dict[str, object]]):
+    from rich import box
+    from rich.table import Table
+
+    table = Table(title="Binary Dependencies", box=box.SIMPLE_HEAVY, expand=True)
+    table.add_column("Binary", no_wrap=True, max_width=28)
+    table.add_column("Plugin", no_wrap=True, max_width=24)
+    table.add_column("State", no_wrap=True, width=8)
+    table.add_column("Status", justify="center", no_wrap=True, width=6)
+    table.add_column("Version", no_wrap=True, width=16)
+    table.add_column("Provider", no_wrap=True, width=8)
+    table.add_column("Path", overflow="fold", ratio=1)
+    for row in rows:
+        table.add_row(
+            str(row["binary"]),
+            str(row["plugin"]),
+            str(row["state"]),
+            str(row["status"]),
+            str(row["version"]),
+            str(row["provider"]),
+            row["path"],
+            style=str(row.get("style") or ""),
+        )
+    return table
+
+
+def _print_binary_row(prnt, row: dict[str, object]) -> None:
+    prnt(
+        "",
+        str(row["binary"]).ljust(28),
+        row["status"],
+        str(row["plugin"]).ljust(24),
+        str(row["state"]).ljust(8),
+        str(row["version"]).ljust(16),
+        str(row["provider"]).ljust(8),
+        row["path"],
+        overflow="ignore",
+        crop=False,
+        style=str(row.get("style") or ""),
+    )
+
+
+def _binary_record_matches_runtime(installed, lib_dir: Path) -> bool:
+    if not installed or not installed.is_valid or not installed.version:
+        return False
+    try:
+        abspath = Path(installed.abspath).expanduser().resolve(strict=False)
+        if not abspath.exists():
+            return False
+        if installed.binprovider not in {"", "env", "apt", "brew"}:
+            abspath.relative_to(lib_dir)
+    except (OSError, ValueError):
+        return False
+    return True
+
+
+def _binary_row_dedupe_key(
+    *,
+    display_name: str,
+    valid: bool,
+    version: str,
+    provider: str,
+    abspath: str,
+) -> tuple[str, str, str, str]:
+    if not valid:
+        return (display_name, "", "", "")
+    try:
+        resolved_abspath = Path(abspath).expanduser().resolve(strict=False).as_posix()
+    except Exception:
+        resolved_abspath = abspath
+    return (display_name, provider, version, resolved_abspath)
+
+
 @enforce_types
 def version(
     quiet: bool = False,
@@ -95,13 +166,14 @@ def version(
 
     from rich.panel import Panel
     from rich.console import Console
+    from rich.live import Live
 
     from archivebox.config import CONSTANTS
     from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
     from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID, IN_DOCKER
     from archivebox.config.paths import get_data_locations, get_code_locations
     from archivebox.misc.logging_util import printable_folder_status
-    from archivebox.config.common import get_config
+    from archivebox.config.common import get_config, normalize_runtime_config
 
     console = Console()
     prnt = console.print
@@ -134,7 +206,7 @@ def version(
     try:
         DATA_DIR_STAT = CONSTANTS.DATA_DIR.stat()
         prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
             f"FS_UID={DATA_DIR_STAT.st_uid}:{DATA_DIR_STAT.st_gid}",
             f"FS_PERMS={config.OUTPUT_PERMISSIONS}",
             f"FS_ATOMIC={config.ENFORCE_ATOMIC_WRITES}",
@@ -142,7 +214,7 @@ def version(
         )
     except Exception:
         prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
         )
 
     prnt(
@@ -155,7 +227,13 @@ def version(
     )
     prnt()
 
-    if not (os.access(config.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
+    in_data_dir = os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)
+    if isinstance(binaries, str):
+        requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
+    else:
+        requested_names = {name for name in (binaries or ()) if name}
+
+    if not in_data_dir:
         PANEL_TEXT = "\n".join(
             (
                 "",
@@ -175,93 +253,219 @@ def version(
                 subtitle="Full version info is only available when inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
             ),
         )
+    prnt()
+
+    if not in_data_dir and not requested_names:
+        prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
+        prnt("", "[grey53]Dependency checks require an initialized collection DATA_DIR.[/grey53]")
+        prnt()
+        prnt("[deep_sky_blue3][i] Code locations:[/deep_sky_blue3]")
+        try:
+            for name, path in get_code_locations().items():
+                if isinstance(name, str) and isinstance(path, dict):
+                    prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
+        except Exception as e:
+            prnt(f"  [red]Error getting code locations: {e}[/red]")
+        prnt()
+        prnt("[red][i] Data locations:[/red] (not in a data directory)")
         prnt()
         return []
 
     prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
     failures = []
+    seen_failures: set[str] = set()
+    seen_rows: set[tuple[str, str, str, str]] = set()
 
-    # Setup Django before importing models
+    from archivebox.plugins.discovery import get_enabled_plugins
+    from abx_dl.config import get_required_binary_requests
+    from abx_dl.dependencies import load_binary
+    from abx_dl.models import discover_plugins, filter_plugins
+
+    plugins = discover_plugins(runtime="archivebox")
+    enabled_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
+    enabled_plugin_names = set(enabled_plugins)
+    runtime_config = normalize_runtime_config(config.for_crawl(), json_safe=False)
+    derived_config: dict[str, object] = {}
+    db_binaries = {}
+    db_available = False
+    if in_data_dir:
+        try:
+            from archivebox.config.django import setup_django
+
+            setup_django()
+
+            from archivebox.machine.models import Machine, Binary
+
+            machine = Machine.current()
+            derived_config = normalize_runtime_config(machine.config, json_safe=False)
+            for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
+                if _binary_record_matches_runtime(binary, config.ABXPKG_LIB_DIR):
+                    db_binaries.setdefault(binary.name, binary)
+            db_available = True
+
+        except Exception as e:
+            prnt()
+            prnt("", f"[yellow]Warning: Could not query collection binary records, falling back to abxpkg state: {e}[/yellow]")
+
+    rows: list[dict[str, object]] = []
+    any_rows = False
+    any_available = False
+    compact_paths = console.is_terminal
+    live_enabled = console.is_terminal
+    live_cm = Live(_build_binary_table(rows), console=console, refresh_per_second=8) if live_enabled else None
+    if not live_enabled:
+        prnt("", "Binary".ljust(28), "Status", "Plugin".ljust(24), "State".ljust(8), "Version".ljust(16), "Provider".ljust(8), "Path")
+
+    def emit_row(row: dict[str, object]) -> None:
+        rows.append(row)
+        if live_cm is not None:
+            live_cm.update(_build_binary_table(rows), refresh=True)
+        else:
+            _print_binary_row(prnt, row)
+
+    if live_cm is not None:
+        live_cm.start()
     try:
-        from archivebox.config.django import setup_django
-
-        setup_django()
-
-        from archivebox.machine.models import Machine, Binary
-
-        machine = Machine.current()
-
-        if isinstance(binaries, str):
-            requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
-        else:
-            requested_names = {name for name in (binaries or ()) if name}
-
-        db_binaries: dict[str, Binary] = {}
-        for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
-            db_binaries.setdefault(binary.name, binary)
-
-        all_binary_names = sorted(requested_names or set(db_binaries.keys()))
-
-        if not all_binary_names:
-            prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
-        else:
-            any_available = False
-            compact_paths = console.is_terminal
-            for name in all_binary_names:
-                if requested_names and name not in requested_names:
+        for plugin_name, plugin in plugins.items():
+            plugin_enabled = plugin_name in enabled_plugin_names
+            logical_records = get_required_binary_requests(
+                plugin,
+                plugin.config.required_binaries,
+                overrides=runtime_config,
+                derived_overrides=derived_config,
+                run_output_dir=CONSTANTS.DATA_DIR,
+            )
+            actual_records = get_required_binary_requests(
+                plugin,
+                plugin.config.required_binaries,
+                overrides=runtime_config,
+                derived_overrides=derived_config,
+                run_output_dir=CONSTANTS.DATA_DIR,
+                logical_names=False,
+            )
+            for logical_record, actual_record in zip(logical_records, actual_records, strict=False):
+                logical_name = str(logical_record["name"])
+                actual_name = str(actual_record["name"])
+                display_name = Path(actual_name).expanduser().name if ("/" in actual_name or actual_name.startswith("~")) else logical_name
+                if (
+                    requested_names
+                    and logical_name not in requested_names
+                    and actual_name not in requested_names
+                    and display_name not in requested_names
+                ):
                     continue
 
-                installed = db_binaries.get(name)
-                if installed and installed.is_valid:
-                    display_name = Path(name).expanduser().name if ("/" in name or name.startswith("~")) else name
+                any_rows = True
+                installed = db_binaries.get(logical_name) if db_available else None
+                if _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
+                    abspath = installed.abspath
+                    version_str = (installed.version or "unknown")[:15]
+                    provider = (installed.binprovider or "env")[:8]
+                    valid = True
+                else:
+                    loaded = load_binary(actual_record)
+                    abspath = str(loaded.loaded_abspath or "")
+                    version_str = str(loaded.loaded_version or "unknown")[:15]
+                    provider = (loaded.loaded_binprovider.name if loaded.loaded_binprovider else "env")[:8]
+                    valid = loaded.is_valid
+
+                if valid:
                     display_path = (
                         _format_binary_abspath(
-                            installed.abspath,
+                            abspath,
                             pwd=Path.cwd(),
-                            lib_dir=config.LIB_DIR,
-                            personas_dir=config.PERSONAS_DIR,
+                            lib_dir=config.ABXPKG_LIB_DIR,
+                            personas_dir=CONSTANTS.PERSONAS_DIR,
                             home=Path.home(),
                         )
                         if compact_paths
-                        else installed.abspath
+                        else abspath
                     )
                     rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
-                    version_str = (installed.version or "unknown")[:15]
-                    provider = (installed.binprovider or "env")[:8]
-                    prnt(
-                        "",
-                        "[green]√[/green]",
-                        "",
-                        display_name.ljust(18),
-                        version_str.ljust(16),
-                        provider.ljust(8),
-                        rendered_path,
-                        overflow="ignore",
-                        crop=False,
-                    )
+                    status = "[green]√[/green]" if plugin_enabled else "[grey53]-[/grey53]"
                     any_available = True
-                    continue
+                else:
+                    rendered_path = "[grey53]not installed[/grey53]"
+                    status = "[red]X[/red]" if plugin_enabled else "[grey53]-[/grey53]"
+                    if plugin_enabled and display_name not in seen_failures:
+                        failures.append(display_name)
+                        seen_failures.add(display_name)
 
-                status = (
-                    "[grey53]not recorded[/grey53]" if name in requested_names and installed is None else "[grey53]not installed[/grey53]"
+                row_key = _binary_row_dedupe_key(
+                    display_name=display_name,
+                    valid=valid,
+                    version=version_str if valid else "-",
+                    provider=provider if valid else "-",
+                    abspath=abspath,
                 )
-                prnt("", "[red]X[/red]", "", name.ljust(18), status, overflow="ignore", crop=False)
-                failures.append(name)
+                if row_key in seen_rows:
+                    continue
+                seen_rows.add(row_key)
 
-            if not any_available:
-                prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
+                emit_row(
+                    {
+                        "plugin": plugin_name,
+                        "state": "enabled" if plugin_enabled else "disabled",
+                        "status": status,
+                        "binary": display_name,
+                        "version": version_str if valid else "-",
+                        "provider": provider if valid else "-",
+                        "path": rendered_path,
+                        "style": "" if plugin_enabled else "dim",
+                    },
+                )
 
-        # Show hint if no binaries are installed yet
-        has_any_installed = Binary.objects.filter(machine=machine).exclude(abspath="").exists()
-        if not has_any_installed:
-            prnt()
-            prnt("", "[grey53]Run [green]archivebox install[/green] to detect and install dependencies.[/grey53]")
+        if db_available:
+            for binary_name, installed in db_binaries.items():
+                if requested_names and binary_name not in requested_names:
+                    continue
+                abspath = installed.abspath
+                version_str = (installed.version or "unknown")[:15]
+                provider = (installed.binprovider or "env")[:8]
+                row_key = _binary_row_dedupe_key(
+                    display_name=binary_name,
+                    valid=True,
+                    version=version_str,
+                    provider=provider,
+                    abspath=abspath,
+                )
+                if row_key in seen_rows:
+                    continue
+                seen_rows.add(row_key)
 
-    except Exception as e:
-        # Handle database errors gracefully (locked, missing, etc.)
-        prnt()
-        prnt("", f"[yellow]Warning: Could not query binaries from database: {e}[/yellow]")
-        prnt("", "[grey53]Run [green]archivebox init[/green] and [green]archivebox install[/green] to set up dependencies.[/grey53]")
+                display_path = (
+                    _format_binary_abspath(
+                        abspath,
+                        pwd=Path.cwd(),
+                        lib_dir=config.ABXPKG_LIB_DIR,
+                        personas_dir=CONSTANTS.PERSONAS_DIR,
+                        home=Path.home(),
+                    )
+                    if compact_paths
+                    else abspath
+                )
+                emit_row(
+                    {
+                        "plugin": "(database)",
+                        "state": "detected",
+                        "status": "[green]√[/green]",
+                        "binary": binary_name,
+                        "version": version_str,
+                        "provider": provider,
+                        "path": _render_binary_abspath(display_path) if compact_paths else display_path,
+                        "style": "dim",
+                    },
+                )
+                any_rows = True
+                any_available = True
+    finally:
+        if live_cm is not None:
+            live_cm.stop()
+
+    if not any_rows:
+        prnt("", "[grey53]No required binaries declared for discovered plugins.[/grey53]")
+    elif not any_available:
+        prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
 
     if not binaries:
         # Show code and data locations
@@ -275,7 +479,7 @@ def version(
             prnt(f"  [red]Error getting code locations: {e}[/red]")
 
         prnt()
-        if os.access(config.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
+        if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
             prnt("[bright_yellow][i] Data locations:[/bright_yellow]")
             try:
                 for name, path in get_data_locations().items():
@@ -320,9 +524,7 @@ def version(
 )
 @docstring(version.__doc__)
 def main(**kwargs):
-    failures = version(**kwargs)
-    if failures:
-        raise SystemExit(1)
+    version(**kwargs)
 
 
 if __name__ == "__main__":

archivebox/config/collection.py

@@ -3,6 +3,7 @@ __package__ = "archivebox.config"
 import io
 import json
 import os
+from collections.abc import Mapping
 from typing import Any
 
 from archivebox.config.constants import CONSTANTS
@@ -54,7 +55,7 @@ def _coerce_to_str_dict(config: Any) -> dict[str, str]:
     """
     if not config:
         return {}
-    if not hasattr(config, "items"):
+    if not isinstance(config, Mapping):
         return {}
     flat: dict[str, str] = {}
     for key, value in config.items():
@@ -98,7 +99,7 @@ def _resolve_section_for_key(key: str, config_sections, plugin_configs) -> str:
 def _render_config_file_content(config: dict[str, str]) -> str:
     """Render a flat config dict to INI text, grouped by inferred section."""
     from archivebox.config.common import get_all_configs
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
 
     config_sections = get_all_configs()
     plugin_configs = discover_plugin_configs()
@@ -284,7 +285,7 @@ def write_config_file(config: dict[str, str]) -> AttrDict:
     """
 
     from archivebox.config.common import get_all_configs
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
     from archivebox.misc.system import atomic_write
 
     config_path = CONSTANTS.CONFIG_FILE

archivebox/config/common.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 __package__ = "archivebox.config"
 
 import json
@@ -6,14 +8,16 @@ import re
 import secrets
 import sys
 import shutil
+import inspect
 from functools import lru_cache
 from collections.abc import Mapping
 from datetime import timedelta
 from typing import Any, ClassVar, cast
 from pathlib import Path
+from urllib.parse import quote
 
 from rich.console import Console
-from pydantic import BaseModel, Field, create_model, field_validator, model_validator
+from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator
 from pydantic_settings import SettingsConfigDict
 from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs
 
@@ -28,6 +32,7 @@ from .permissions import IN_DOCKER
 ConfigOverrides = Mapping[str, object]
 ConfigPayload = dict[str, object]
 PluginSchemaDocuments = dict[str, dict[str, Any]]
+LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 
 ###################### Config ##########################
 
@@ -62,8 +67,30 @@ def permissions_from_legacy_public_flags(raw_config: Mapping[str, object]) -> st
     return None
 
 
+def resolve_delete_after_config_value(*configs: Mapping[str, Any] | None) -> str:
+    for config in configs:
+        if config is None:
+            continue
+        value = config.get("DELETE_AFTER")
+        if value:
+            return str(value)
+    return "0"
+
+
 _SENSITIVE_CONFIG_KEY_NEEDLES = ("TOKEN", "SECRET", "API_KEY", "APIKEY", "PASSWORD")
 SENSITIVE_CONFIG_VALUE_REDACTED = "********"
+_SCOPE_CRAWL_FROZEN = "crawl_frozen"
+_SCOPE_CRAWL_EXECUTION = "crawl_execution"
+_SCOPE_SERVER = "server"
+
+
+@lru_cache(maxsize=1)
+def _plugin_sensitive_config_keys() -> frozenset[str]:
+    sensitive_keys: set[str] = set()
+    for prop_key, prop_schema in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).items():
+        if isinstance(prop_schema, Mapping) and prop_schema.get("x-sensitive"):
+            sensitive_keys.add(str(prop_key))
+    return frozenset(sensitive_keys)
 
 
 def is_sensitive_config_key(key: str) -> bool:
@@ -77,8 +104,9 @@ def is_sensitive_config_key(key: str) -> bool:
     REST API responses, and any future surface that round-trips raw config
     values all agree on which keys to redact.
     """
-    upper = (key or "").upper()
-    return any(needle in upper for needle in _SENSITIVE_CONFIG_KEY_NEEDLES)
+    key = str(key or "")
+    upper = key.upper()
+    return key in _plugin_sensitive_config_keys() or any(needle in upper for needle in _SENSITIVE_CONFIG_KEY_NEEDLES)
 
 
 def redact_sensitive_config(config: Mapping[str, Any] | None) -> dict[str, Any]:
@@ -103,6 +131,79 @@ def redact_sensitive_config(config: Mapping[str, Any] | None) -> dict[str, Any]:
     return redacted
 
 
+def normalize_runtime_config(
+    config: BaseConfigSet | Mapping[str, Any] | str | None,
+    *,
+    only_crawl_execution: bool = False,
+    exclude_runtime_derived: bool = False,
+    exclude_crawl_execution: bool = False,
+    json_safe: bool = True,
+) -> dict[str, Any]:
+    """Return config filtered for runtime/frozen usage, optionally JSON-safe."""
+    if config is None:
+        return {}
+    if isinstance(config, BaseConfigSet):
+        config = config.model_dump(mode="json")
+    elif isinstance(config, str):
+        config = json.loads(config)
+    else:
+        config = dict(config)
+
+    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys() if exclude_runtime_derived else frozenset()
+    filtered = {
+        key: value
+        for key, value in config.items()
+        if (
+            value is not None
+            and (not only_crawl_execution or ArchiveBoxConfig.scope_for_key(str(key)) == _SCOPE_CRAWL_EXECUTION)
+            and (not exclude_runtime_derived or str(key) not in runtime_derived_keys)
+            and (not exclude_crawl_execution or ArchiveBoxConfig.scope_for_key(str(key)) != _SCOPE_CRAWL_EXECUTION)
+        )
+    }
+    if not json_safe:
+        return filtered
+    return {key: value for key, value in json.loads(json.dumps(filtered, default=str)).items() if value is not None}
+
+
+def build_crawl_config_snapshot(
+    *,
+    persona: Any = None,
+    overrides: Mapping[str, Any] | None = None,
+    base_config: ArchiveBoxBaseConfig | Mapping[str, object] | None = None,
+) -> dict[str, Any]:
+    """Build the frozen crawl config stored on Crawl.config at creation time."""
+    explicit_overrides = set(overrides or {})
+    plugin_owned_keys = set(_plugin_config_properties(PLUGIN_CONFIG_SCHEMAS)) - set(ArchiveBoxBaseConfig.model_fields)
+    effective = get_config(persona=persona, base_config=base_config)
+    frozen = effective.for_crawl_frozen(persona=persona)
+    for key in ("BIND_ADDR", "BASE_URL", "CSRF_TRUSTED_ORIGINS", "SERVER_SECURITY_MODE"):
+        value = getattr(effective, key, None)
+        if value is not None:
+            frozen[key] = value
+    if persona is not None:
+        persona_config = persona.get_derived_config()
+        for key in plugin_owned_keys - explicit_overrides:
+            if key in persona_config:
+                frozen.pop(key, None)
+    if overrides:
+        resolved = get_config(base_config=frozen, overrides=overrides, include_machine=False)
+        resolved_payload = normalize_runtime_config(resolved)
+        frozen = resolved.for_crawl_frozen(persona=persona)
+        for key in ("BIND_ADDR", "BASE_URL", "CSRF_TRUSTED_ORIGINS", "SERVER_SECURITY_MODE"):
+            value = getattr(resolved, key, None)
+            if value is not None:
+                frozen[key] = value
+        for key in plugin_owned_keys & explicit_overrides:
+            if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_FROZEN and key in resolved_payload:
+                frozen[key] = resolved_payload[key]
+        if persona is not None:
+            persona_config = persona.get_derived_config()
+            for key in plugin_owned_keys - explicit_overrides:
+                if key in persona_config:
+                    frozen.pop(key, None)
+    return frozen
+
+
 def rprint(*args, file=None, **kwargs):
     console = _STDERR_CONSOLE if file is sys.stderr else _STDOUT_CONSOLE
     console.print(*args, **kwargs)
@@ -110,6 +211,7 @@ def rprint(*args, file=None, **kwargs):
 
 class ShellConfig(BaseConfigSet):
     toml_section_header: str = "SHELL_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_CRAWL_EXECUTION)
 
     DEBUG: bool = Field(default="--debug" in sys.argv)
 
@@ -141,29 +243,17 @@ class ShellConfig(BaseConfigSet):
 
 class StorageConfig(BaseConfigSet):
     toml_section_header: str = "STORAGE_CONFIG"
-
-    # ARCHIVE_DIR / USERS_DIR are resolved dynamically via get_config().
-    ARCHIVE_DIR: Path = Field(default=CONSTANTS.ARCHIVE_DIR)
-    USERS_DIR: Path = Field(default=CONSTANTS.USERS_DIR)
-    PERSONAS_DIR: Path = Field(default=CONSTANTS.PERSONAS_DIR)
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     # TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
-    # must be a short path due to unix path length restrictions for socket files (<100 chars)
+    # must be a short path due to unix path length restrictions for socket files (<90 chars)
     # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
-    TMP_DIR: Path = Field(default=CONSTANTS.DEFAULT_TMP_DIR)
+    TMP_DIR: Path = Field(default=CONSTANTS.DEFAULT_TMP_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
-    # LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
+    # ABXPKG_LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
     # must be able to contain executable binaries (up to 5GB size)
     # should not be a remote/network/FUSE mount for speed reasons, otherwise extractors will be slow
-    LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_DIR)
-
-    # LIB_BIN_DIR is where installed binaries can be symlinked for shared runtime lookup.
-    # abxpkg/abx-dl build the executable lookup env at exec time.
-    LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR)
-
-    # CUSTOM_TEMPLATES_DIR allows users to override default templates
-    # defaults to DATA_DIR / 'user_templates' but can be configured
-    CUSTOM_TEMPLATES_DIR: Path = Field(default=CONSTANTS.CUSTOM_TEMPLATES_DIR)
+    ABXPKG_LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_ABXPKG_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
     OUTPUT_PERMISSIONS: str = Field(default="644")
     ENFORCE_ATOMIC_WRITES: bool = Field(default=True)
@@ -172,12 +262,14 @@ class StorageConfig(BaseConfigSet):
 
 class GeneralConfig(BaseConfigSet):
     toml_section_header: str = "GENERAL_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     TAG_SEPARATOR_PATTERN: str = Field(default=r"[,]")
 
 
 class ServerConfig(BaseConfigSet):
     toml_section_header: str = "SERVER_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     SERVER_SECURITY_MODES: ClassVar[tuple[str, ...]] = (
         "safe-subdomains-fullreplay",
@@ -197,8 +289,6 @@ class ServerConfig(BaseConfigSet):
     FOOTER_INFO: str = Field(
         default="Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.",
     )
-    # CUSTOM_TEMPLATES_DIR: Path          = Field(default=None)  # this is now a constant
-
     PUBLIC_INDEX: bool = Field(default=True)
     PUBLIC_ADD_VIEW: bool = Field(default=False)
 
@@ -258,6 +348,7 @@ class ServerConfig(BaseConfigSet):
 
 class DatabaseConfig(BaseConfigSet):
     toml_section_header: str = "DATABASE_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     DATABASE_NAME: str = Field(default=str(CONSTANTS.DATABASE_FILE), alias="ARCHIVEBOX_DATABASE_NAME")
     SQLITE_JOURNAL_MODE: str = Field(
@@ -277,6 +368,7 @@ class DatabaseConfig(BaseConfigSet):
 
 class ArchivingConfig(BaseConfigSet):
     toml_section_header: str = "ARCHIVING_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_CRAWL_FROZEN)
 
     PLUGINS: str = Field(
         default="",
@@ -284,6 +376,7 @@ class ArchivingConfig(BaseConfigSet):
     )
 
     ONLY_NEW: bool = Field(default=True)
+    INDEX_ONLY: bool = Field(default=False)
 
     TIMEOUT: int = Field(default=60)
     CRAWL_MAX_URLS: int = Field(default=0)
@@ -305,7 +398,7 @@ class ArchivingConfig(BaseConfigSet):
     URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST")
     URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST")
 
-    DEFAULT_PERSONA: str = Field(default="Default")
+    DEFAULT_PERSONA: str = Field(default="Default", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     PERMISSIONS: str = Field(
         default="public",
         description="Snapshot visibility: public lists and serves content, unlisted serves direct links only, private requires admin login.",
@@ -397,8 +490,9 @@ def parse_delete_after(value) -> timedelta | None:
 
 class SearchBackendConfig(BaseConfigSet):
     toml_section_header: str = "SEARCH_BACKEND_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
-    SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep")
+    SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
 
 def _plugin_user_config_value(value: Any) -> str:
@@ -414,13 +508,11 @@ def _plugin_user_config(config: Mapping[str, object]) -> dict[str, str]:
 
 
 def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
 
     schemas: PluginSchemaDocuments = {}
     if BASE_CONFIG_PATH.exists():
-        schemas["base"] = {
-            "properties": json.loads(BASE_CONFIG_PATH.read_text()).get("properties", {}),
-        }
+        schemas["base"] = json.loads(BASE_CONFIG_PATH.read_text())
     schemas.update(discover_plugin_configs())
     return schemas
 
@@ -444,6 +536,10 @@ def _archivebox_config_input_names() -> set[str]:
     for field in ArchiveBoxConfig.model_fields.values():
         if isinstance(field.alias, str):
             names.add(field.alias)
+    for prop in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).values():
+        aliases = prop.get("x-aliases") if isinstance(prop, Mapping) else None
+        if isinstance(aliases, list):
+            names.update(str(alias) for alias in aliases if str(alias).strip())
     return names
 
 
@@ -473,42 +569,197 @@ class ArchiveBoxBaseConfig(
         populate_by_name=True,
     )
 
-    DATA_DIR: Path = Field(default=CONSTANTS.DATA_DIR)
-    ABX_RUNTIME: str = Field(default="archivebox")
-    CRAWL_DIR: Path | None = Field(default=None)
-    SNAP_DIR: Path | None = Field(default=None)
     computed_config_keys: ClassVar[tuple[str, ...]] = COMPUTED_CONFIG_KEYS
 
+    @classmethod
+    def _core_config_classes(cls) -> tuple[type[BaseConfigSet], ...]:
+        return (
+            ShellConfig,
+            StorageConfig,
+            GeneralConfig,
+            ServerConfig,
+            DatabaseConfig,
+            ArchivingConfig,
+            SearchBackendConfig,
+            LDAPConfig,
+        )
+
+    @classmethod
+    def _core_field_scope(cls, key: str) -> str | None:
+        if key == "toml_section_header":
+            return _SCOPE_SERVER
+        for config_cls in cls._core_config_classes():
+            field = config_cls.model_fields.get(key)
+            if field is None:
+                continue
+            default_scope = str(config_cls.__private_attributes__["_scope"].default)
+            extra = field.json_schema_extra
+            if isinstance(extra, dict) and "scope" in extra:
+                return str(extra["scope"])
+            return default_scope
+        if key in ArchiveBoxBaseConfig.model_fields:
+            field = ArchiveBoxBaseConfig.model_fields[key]
+            extra = field.json_schema_extra
+            if isinstance(extra, dict) and "scope" in extra:
+                return str(extra["scope"])
+            return _SCOPE_SERVER
+        return None
+
+    @classmethod
+    def _plugin_field_scope(cls, key: str) -> str | None:
+        scope = None
+        for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+            properties = schema.get("properties") if isinstance(schema, dict) else None
+            if not isinstance(properties, dict) or key not in properties:
+                continue
+            prop_schema = properties.get(key) or {}
+            if isinstance(prop_schema, Mapping) and prop_schema.get("x-scope"):
+                scope = str(prop_schema["x-scope"])
+            elif scope is None:
+                scope = _SCOPE_CRAWL_FROZEN
+        return scope
+
+    @classmethod
+    @lru_cache(maxsize=None)
+    def scope_for_key(cls, key: str) -> str:
+        for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+            properties = schema.get("properties") if isinstance(schema, dict) else None
+            if isinstance(properties, dict) and key == f"{str(plugin_name).upper()}_ENABLED" and key in properties:
+                return _SCOPE_CRAWL_EXECUTION
+        if key.endswith("_BINARY"):
+            return _SCOPE_CRAWL_EXECUTION
+        return cls._core_field_scope(key) or cls._plugin_field_scope(key) or _SCOPE_SERVER
+
+    @classmethod
+    @lru_cache(maxsize=1)
+    def _scope_by_key(cls) -> dict[str, str]:
+        return {key: cls.scope_for_key(key) for key in cls.model_fields}
+
+    @classmethod
+    @lru_cache(maxsize=1)
+    def _crawl_frozen_keys(cls) -> frozenset[str]:
+        return frozenset(key for key, scope in cls._scope_by_key().items() if scope == _SCOPE_CRAWL_FROZEN)
+
+    @classmethod
+    @lru_cache(maxsize=1)
+    def _crawl_runtime_keys(cls) -> frozenset[str]:
+        return frozenset(key for key, scope in cls._scope_by_key().items() if scope in {_SCOPE_CRAWL_FROZEN, _SCOPE_CRAWL_EXECUTION})
+
+    @classmethod
+    @lru_cache(maxsize=1)
+    def runtime_derived_config_keys(cls) -> frozenset[str]:
+        runtime_derived_keys = {
+            "ABX_INSTALL_CACHE",
+            "ACTIVE_PERSONA",
+            "CRAWL_DIR",
+            "DEFAULT_PERSONA",
+            "EXTRA_CONTEXT",
+            "PERSONAS_DIR",
+            "SNAP_DIR",
+        }
+        return frozenset(
+            key for key, scope in cls._scope_by_key().items() if scope == _SCOPE_CRAWL_EXECUTION and key in runtime_derived_keys
+        )
+
+    def _scoped_config(self, *, include_execution: bool) -> dict[str, Any]:
+        keys = type(self)._crawl_runtime_keys() if include_execution else type(self)._crawl_frozen_keys()
+        payload = self.model_dump(mode="json")
+        return {key: payload[key] for key in keys if payload.get(key, None) is not None}
+
+    def for_crawl(self) -> dict[str, Any]:
+        """Config scoped to crawl execution, without runtime object overlays."""
+        config = self._scoped_config(include_execution=True)
+        for key in type(self).runtime_derived_config_keys():
+            config.pop(key, None)
+        return config
+
+    def for_crawl_frozen(self, *, persona: Any = None) -> dict[str, Any]:
+        """Config safe to persist permanently on Crawl.config."""
+        frozen = self._scoped_config(include_execution=False)
+        if persona is not None:
+            persona_config = dict(persona.config or {})
+            scope_by_key = type(self)._scope_by_key()
+            for key in persona.get_derived_config():
+                if key not in persona_config and scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION:
+                    frozen.pop(key, None)
+        return frozen
+
+    def for_crawl_runtime(
+        self,
+        *,
+        crawl: Any = None,
+        snapshot: Any = None,
+        persona: Any = None,
+        runtime_overrides: Mapping[str, Any] | None = None,
+        extra_context: Mapping[str, Any] | None = None,
+        crawl_output_dir: Any = None,
+        snapshot_output_dir: Any = None,
+    ) -> dict[str, Any]:
+        """Config payload safe to pass to crawl/snapshot hook execution."""
+        config = self.for_crawl()
+        config["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
+        scope_by_key = type(self)._scope_by_key()
+        model_fields = type(self).model_fields
+        # ArchiveBox-only selectors are consumed during model validation to
+        # derive hook-owned *_ENABLED flags. They are stripped below after
+        # crawl/snapshot overlays have had a chance to apply.
+        config.pop("SEARCH_BACKEND_ENGINE", None)
+        if persona is not None:
+            for key, value in persona.get_derived_config().items():
+                if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION:
+                    config[key] = value
+
+        if crawl is not None:
+            for key, value in dict(crawl.config or {}).items():
+                if key in model_fields and scope_by_key.get(key) != _SCOPE_CRAWL_EXECUTION:
+                    config[key] = value
+            config["CRAWL_DIR"] = str(crawl_output_dir if crawl_output_dir is not None else crawl.output_dir)
+
+        if snapshot is not None:
+            for key, value in dict(snapshot.config or {}).items():
+                if key in model_fields and scope_by_key.get(key) != _SCOPE_CRAWL_EXECUTION:
+                    config[key] = value
+            config["SNAP_DIR"] = str(snapshot_output_dir if snapshot_output_dir is not None else snapshot.output_dir)
+
+        if runtime_overrides:
+            config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
+
+        # Hooks should only see concrete plugin-local flags, never the
+        # ArchiveBox selectors used to derive them.
+        config.pop("PLUGINS", None)
+        config.pop("SEARCH_BACKEND_ENGINE", None)
+
+        if extra_context:
+            context: dict[str, Any] = {}
+            if config.get("EXTRA_CONTEXT"):
+                parsed_extra_context = json.loads(str(config["EXTRA_CONTEXT"]))
+                if not isinstance(parsed_extra_context, dict):
+                    raise TypeError("EXTRA_CONTEXT must decode to an object")
+                context = parsed_extra_context
+            context.update(dict(extra_context))
+            config["EXTRA_CONTEXT"] = json.dumps(context, separators=(",", ":"), sort_keys=True)
+
+        return config
+
     @model_validator(mode="after")
     def resolve_runtime_paths(self):
-        self.DATA_DIR = self.DATA_DIR.expanduser().resolve()
-
-        archive_dir = self.ARCHIVE_DIR.expanduser()
-        if archive_dir == (CONSTANTS.DATA_DIR / CONSTANTS.ARCHIVE_DIR_NAME) and self.DATA_DIR != CONSTANTS.DATA_DIR:
-            archive_dir = self.DATA_DIR / CONSTANTS.ARCHIVE_DIR_NAME
-        if not archive_dir.is_absolute():
-            archive_dir = self.DATA_DIR / archive_dir
-        self.ARCHIVE_DIR = archive_dir.resolve()
-
-        users_dir = self.USERS_DIR.expanduser()
-        if users_dir == (CONSTANTS.ARCHIVE_DIR / CONSTANTS.USERS_DIR_NAME):
-            users_dir = self.ARCHIVE_DIR / CONSTANTS.USERS_DIR_NAME
-        if not users_dir.is_absolute():
-            users_dir = self.ARCHIVE_DIR / users_dir
-        self.USERS_DIR = users_dir.resolve()
-
-        lib_dir = self.LIB_DIR.expanduser()
+        lib_dir = self.ABXPKG_LIB_DIR.expanduser()
         if not lib_dir.is_absolute():
-            lib_dir = self.DATA_DIR / lib_dir
-        self.LIB_DIR = lib_dir.resolve()
+            lib_dir = CONSTANTS.DATA_DIR / lib_dir
+        self.ABXPKG_LIB_DIR = lib_dir.resolve()
 
-        lib_bin_dir = self.LIB_BIN_DIR.expanduser()
-        if lib_bin_dir == CONSTANTS.DEFAULT_LIB_BIN_DIR and self.LIB_DIR != CONSTANTS.DEFAULT_LIB_DIR:
-            lib_bin_dir = self.LIB_DIR / "bin"
-        elif not lib_bin_dir.is_absolute():
-            lib_bin_dir = self.DATA_DIR / lib_bin_dir
-        self.LIB_BIN_DIR = lib_bin_dir.resolve()
+        return self
 
+    @model_validator(mode="after")
+    def derive_plugin_enabled_config(self):
+        plugin_names = _normalize_plugins_config_value(self.PLUGINS)
+        selected_plugins = _plugins_with_required_plugins(plugin_names) if plugin_names else set()
+        search_backend = self.SEARCH_BACKEND_ENGINE.strip().lower()
+        if search_backend:
+            selected_plugins.add(f"search_backend_{search_backend}")
+        for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
+            if plugin_names or plugin_name in selected_plugins:
+                setattr(self, enabled_key, plugin_name in selected_plugins)
         return self
 
 
@@ -532,41 +783,213 @@ PLUGIN_CONFIG_SCHEMAS = _discover_plugin_config_schemas()
 ArchiveBoxConfig = _build_archivebox_config_model(PLUGIN_CONFIG_SCHEMAS)
 
 
+def _normalize_plugins_config_value(value: Any) -> set[str]:
+    if value is None:
+        return set()
+    if isinstance(value, str):
+        raw = value.strip()
+        if not raw:
+            return set()
+        if raw.startswith("["):
+            try:
+                parsed = json.loads(raw)
+            except json.JSONDecodeError:
+                parsed = None
+            if isinstance(parsed, list):
+                return {str(plugin).strip().lower() for plugin in parsed if str(plugin).strip()}
+        return {plugin.strip().lower() for plugin in raw.split(",") if plugin.strip()}
+    if isinstance(value, (list, tuple, set)):
+        return {str(plugin).strip().lower() for plugin in value if str(plugin).strip()}
+    normalized = str(value).strip().lower()
+    return {normalized} if normalized else set()
+
+
+@lru_cache(maxsize=1)
+def _plugin_enabled_config_keys() -> dict[str, str]:
+    enabled_keys: dict[str, str] = {}
+    for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+        properties = schema.get("properties") if isinstance(schema, dict) else None
+        if not isinstance(properties, dict):
+            continue
+        enabled_key = f"{str(plugin_name).upper()}_ENABLED"
+        if enabled_key in properties and ArchiveBoxConfig.scope_for_key(enabled_key) == _SCOPE_CRAWL_EXECUTION:
+            enabled_keys[str(plugin_name).lower()] = enabled_key
+    return enabled_keys
+
+
+def _plugins_with_required_plugins(plugin_names: set[str]) -> set[str]:
+    selected = set(plugin_names)
+    pending = list(selected)
+    while pending:
+        plugin_name = pending.pop()
+        schema = PLUGIN_CONFIG_SCHEMAS.get(plugin_name, {})
+        required_plugins = schema.get("required_plugins") if isinstance(schema, dict) else None
+        if not isinstance(required_plugins, list):
+            continue
+        for required_plugin in required_plugins:
+            required_plugin_name = str(required_plugin).strip().lower()
+            if required_plugin_name and required_plugin_name not in selected:
+                selected.add(required_plugin_name)
+                pending.append(required_plugin_name)
+    return selected
+
+
+def get_live_config_url(key: str) -> str:
+    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
+
+
+@lru_cache(maxsize=1)
+def config_field_metadata() -> dict[str, dict[str, Any]]:
+    """Return one centralized metadata map for core and plugin config fields."""
+    metadata: dict[str, dict[str, Any]] = {}
+    for key, field in ArchiveBoxConfig.model_fields.items():
+        if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_EXECUTION or key in ArchiveBoxConfig.computed_config_keys:
+            continue
+        default = field.default
+        try:
+            json.dumps(default)
+        except TypeError:
+            default = str(default)
+        metadata[key] = {
+            "plugin": "archivebox",
+            "section": find_config_section(key),
+            "type": config_field_type(key),
+            "default": default,
+            "description": field.description or "",
+            "scope": ArchiveBoxConfig.scope_for_key(key),
+            "sensitive": is_sensitive_config_key(key),
+        }
+    for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+        properties = schema.get("properties") if isinstance(schema, dict) else None
+        if not isinstance(properties, dict):
+            continue
+        for key, prop in properties.items():
+            if not isinstance(prop, Mapping):
+                continue
+            if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_EXECUTION:
+                continue
+            metadata[key] = {
+                **metadata.get(key, {}),
+                "plugin": plugin_name,
+                "section": "PLUGINS",
+                "type": prop.get("type", metadata.get(key, {}).get("type", "string")),
+                "default": prop.get("default", metadata.get(key, {}).get("default", "")),
+                "description": prop.get("description", metadata.get(key, {}).get("description", "")),
+                "scope": ArchiveBoxConfig.scope_for_key(key),
+                "sensitive": bool(prop.get("x-sensitive")) or is_sensitive_config_key(key),
+                "schema": dict(prop),
+            }
+    return metadata
+
+
+def find_config_section(key: str) -> str:
+    from archivebox.config import CONSTANTS_CONFIG
+
+    if key in CONSTANTS_CONFIG:
+        return "CONSTANT"
+    for section_id, section in get_all_configs().items():
+        if key in type(section).model_fields:
+            return section_id
+    if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
+        return "PLUGINS"
+    return "DYNAMIC"
+
+
+def find_config_default(key: str) -> str:
+    from archivebox.config import CONSTANTS_CONFIG
+
+    if key in CONSTANTS_CONFIG:
+        return str(CONSTANTS_CONFIG[key])
+
+    field = ArchiveBoxConfig.model_fields.get(key)
+    if field is None:
+        return ""
+    default_val = field.default
+    if callable(default_val):
+        default_val = inspect.getsource(default_val).split("lambda", 1)[-1].split(":", 1)[-1].replace("\n", " ").strip()
+        if default_val.count(")") > default_val.count("("):
+            default_val = default_val[:-1]
+    else:
+        default_val = str(default_val)
+    return default_val
+
+
+def config_field_type(key: str) -> str:
+    field = ArchiveBoxConfig.model_fields.get(key)
+    if field is None:
+        return "str"
+    annotation = field.annotation
+    try:
+        return annotation.__name__
+    except AttributeError:
+        return str(annotation)
+
+
+def find_config_type(key: str) -> str:
+    return config_field_type(key)
+
+
+def find_config_source(key: str, merged_config: Mapping[str, Any]) -> str:
+    """Determine where a config value comes from."""
+    from archivebox.machine.models import Machine
+
+    machine = Machine.current()
+    if machine.config and key in machine.config:
+        return "Machine"
+
+    if key in os.environ:
+        return "Environment"
+
+    file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
+    if key in file_config:
+        return "File"
+
+    if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
+        return "Plugin Default"
+
+    return "Default"
+
+
+def get_request_config(request: Any, *, resolve_plugins: bool = False) -> ArchiveBoxBaseConfig:
+    """Return the per-request ArchiveBox config, upgrading to plugin resolution if needed."""
+    request_state = request.__dict__
+    request_config = request_state.get("archivebox_config")
+    request_config_resolves_plugins = bool(request_state.get("_archivebox_config_resolves_plugins", False))
+    if request_config is None or (resolve_plugins and not request_config_resolves_plugins):
+        request_config = get_config(resolve_plugins=resolve_plugins)
+        request.archivebox_config = request_config
+        request._archivebox_config_resolves_plugins = resolve_plugins
+    return request_config
+
+
 def get_config(
     defaults: ConfigOverrides | None = None,
     overrides: ConfigOverrides | None = None,
     base_config: ArchiveBoxBaseConfig | Mapping[str, object] | None = None,
     persona: Any = None,
-    user: Any = None,
     crawl: Any = None,
     snapshot: Any = None,
-    archiveresult: Any = None,
     machine: Any = None,
     include_machine: bool = True,
     resolve_plugins: bool = True,
+    redact_sensitive: bool = False,
 ) -> ArchiveBoxBaseConfig:
     """
     Get merged config from all sources.
 
-    Priority (highest to lowest):
-    1. Explicit overrides
-    2. Per-ArchiveResult config
-    3. Per-snapshot config and output path
-    4. Per-crawl config and output path
-    5. Per-user config
-    6. Per-persona derived config
-    7. Current machine derived config
-    8. Environment variables
-    9. Config file (ArchiveBox.conf)
-    10. Plugin schema defaults
-    11. Core config defaults
-    """
-    if snapshot is None and archiveresult is not None:
-        snapshot = archiveresult.snapshot
+    Defaults are hydrated by pydantic from core/plugin defaults,
+    ArchiveBox.conf, and environment variables. Persisted Machine/Persona
+    values then apply for live crawl-execution scope, while Crawl/Snapshot
+    rows apply their frozen crawl-scope values. Explicit overrides win last.
 
+    Crawl-execution config is not persisted on Crawl.config. It is rederived
+    from current Machine/Persona state and hydrated process defaults each time.
+    """
     if crawl is None and snapshot is not None:
         crawl = snapshot.crawl
 
+    crawl_config_base = crawl is not None and base_config is None
+
     if include_machine and machine is None:
         try:
             from django.apps import apps
@@ -582,13 +1005,23 @@ def get_config(
         persona = crawl.resolve_persona()
 
     config_data: ConfigPayload = dict(defaults or {})
-    if base_config is not None:
+    config_data["PERSONAS_DIR"] = str(CONSTANTS.PERSONAS_DIR)
+    base_config_payload: ConfigPayload = {}
+    if crawl_config_base:
+        config_data.update(
+            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+        )
+        config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
+    elif base_config is not None:
         if isinstance(base_config, ArchiveBoxBaseConfig):
-            config_data.update(base_config.model_dump(mode="json"))
+            base_config_payload.update(base_config.model_dump(mode="json"))
         else:
-            config_data.update(dict(base_config))
+            base_config_payload.update(dict(base_config))
+        config_data.update(normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False))
     else:
-        config_data.update(ArchiveBoxConfig().model_dump(mode="json"))
+        config_data.update(
+            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+        )
         legacy_permissions = permissions_from_legacy_public_flags({**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ})
         if legacy_permissions:
             config_data["PERMISSIONS"] = legacy_permissions
@@ -598,33 +1031,39 @@ def get_config(
     if include_machine and machine is not None and machine.config:
         from archivebox.machine.models import _sanitize_machine_config
 
-        scope_overrides.update(_sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")))
+        scope_overrides.update(
+            normalize_runtime_config(
+                _sanitize_machine_config(machine.config, lib_dir=config_data.get("ABXPKG_LIB_DIR")),
+                only_crawl_execution=crawl_config_base,
+                exclude_runtime_derived=True,
+                json_safe=False,
+            ),
+        )
 
     if persona is not None:
-        scope_overrides.update(persona.get_derived_config())
+        persona_config = normalize_runtime_config(
+            persona.get_derived_config(),
+            exclude_runtime_derived=not crawl_config_base,
+            json_safe=False,
+        )
+        if crawl_config_base:
+            scope_by_key = ArchiveBoxConfig._scope_by_key()
+            crawl_keys = set(dict(crawl.config or {}))
+            persona_config = {
+                key: value
+                for key, value in persona_config.items()
+                if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION or key not in crawl_keys
+            }
+        scope_overrides.update(persona_config)
 
-    user_config = getattr(user, "config", None)
-    if user_config:
-        scope_overrides.update(user_config)
-
-    if crawl is not None and crawl.config:
-        scope_overrides.update(crawl.config)
-
-    if crawl is not None:
-        if not overrides or "CRAWL_DIR" not in overrides:
-            scope_overrides["CRAWL_DIR"] = crawl.output_dir
+    if crawl is not None and crawl.config and not crawl_config_base:
+        scope_overrides.update(normalize_runtime_config(crawl.config, exclude_crawl_execution=True, json_safe=False))
 
     if snapshot is not None and snapshot.config:
-        scope_overrides.update(snapshot.config)
-
-    if snapshot is not None:
-        scope_overrides["SNAP_DIR"] = snapshot.output_dir
-
-    if archiveresult is not None and archiveresult.config:
-        scope_overrides.update(archiveresult.config)
+        scope_overrides.update(normalize_runtime_config(snapshot.config, exclude_crawl_execution=True, json_safe=False))
 
     if overrides:
-        scope_overrides.update(overrides)
+        scope_overrides.update(normalize_runtime_config(overrides, exclude_crawl_execution=True, json_safe=False))
 
     legacy_scope_permissions = permissions_from_legacy_public_flags(scope_overrides)
     if legacy_scope_permissions:
@@ -634,21 +1073,46 @@ def get_config(
     config_data.update(archivebox_scope_overrides)
 
     if resolve_plugins:
-        plugin_schemas = {
-            plugin_name: schema.get("properties", {}) for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)
-        }
+        plugin_schemas = {plugin_name: schema for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)}
         plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
+        plugin_user_config = _plugin_user_config(
+            {
+                **normalize_runtime_config(config_data, only_crawl_execution=True, json_safe=False),
+                **scope_overrides,
+            },
+        )
+        if not crawl_config_base:
+            plugin_user_config = {
+                **normalize_runtime_config(
+                    BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE),
+                    exclude_runtime_derived=True,
+                    json_safe=False,
+                ),
+                **plugin_user_config,
+            }
         plugin_sections = resolve_plugin_configs(
             plugin_schemas,
             global_config=plugin_global_config,
-            user_config={**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **_plugin_user_config(scope_overrides)},
+            user_config=plugin_user_config,
+            environ={},
         )
         for plugin_config in plugin_sections.values():
-            config_data.update(plugin_config)
+            for key, value in plugin_config.items():
+                if key in ArchiveBoxBaseConfig.model_fields and key not in archivebox_scope_overrides and key not in base_config_payload:
+                    continue
+                config_data[key] = value
+        if base_config_payload:
+            config_data.update(
+                {
+                    key: value
+                    for key, value in normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False).items()
+                    if key in _archivebox_config_input_names()
+                },
+            )
+        if crawl_config_base:
+            config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
         config_data.update(archivebox_scope_overrides)
 
-    config_data["ABX_RUNTIME"] = "archivebox"
-
     # Decode JSON-encoded complex values (dict/list fields) that came from
     # string-only sources before validation. ``IniConfigSettingsSource`` does
     # this for the ArchiveBox.conf path, but Machine.config (mirrored from the
@@ -673,9 +1137,12 @@ def get_config(
             )
 
     config = ArchiveBoxConfig.model_validate(config_data)
-    os.environ["LIB_DIR"] = str(config.LIB_DIR)
-    os.environ["LIB_BIN_DIR"] = str(config.LIB_BIN_DIR)
-    os.environ["ABXPKG_LIB_DIR"] = str(config.LIB_DIR)
+    if redact_sensitive:
+        for key in type(config).model_fields:
+            value = config[key]
+            if is_sensitive_config_key(key) and value not in (None, ""):
+                setattr(config, key, SENSITIVE_CONFIG_VALUE_REDACTED)
+    os.environ["ABXPKG_LIB_DIR"] = str(config.ABXPKG_LIB_DIR)
     archiving_warning_key = (config.TIMEOUT, config.USE_COLOR)
     if archiving_warning_key not in _WARNED_ARCHIVING_CONFIGS:
         config.warn_if_invalid()

archivebox/config/constants.py

@@ -36,8 +36,8 @@ from .permissions import (
     IN_DOCKER,
     RUNNING_AS_UID,
     RUNNING_AS_GID,
-    DEFAULT_PUID,
-    DEFAULT_PGID,
+    DEFAULT_UID,
+    DEFAULT_GID,
     ARCHIVEBOX_USER,
     ARCHIVEBOX_GROUP,
 )
@@ -66,8 +66,8 @@ class ConstantsDict:
     ARCHIVEBOX_GROUP: int = ARCHIVEBOX_GROUP
     RUNNING_AS_UID: int = RUNNING_AS_UID
     RUNNING_AS_GID: int = RUNNING_AS_GID
-    DEFAULT_PUID: int = DEFAULT_PUID
-    DEFAULT_PGID: int = DEFAULT_PGID
+    DEFAULT_UID: int = DEFAULT_UID
+    DEFAULT_GID: int = DEFAULT_GID
     IS_INSIDE_VENV: bool = sys.prefix != sys.base_prefix
 
     # Source code dirs
@@ -113,12 +113,10 @@ class ConstantsDict:
     TMP_DIR_NAME: str = "tmp"
     DEFAULT_TMP_DIR: Path = DATA_DIR / TMP_DIR_NAME / MACHINE_ID  # ./data/tmp/abc3244323
 
-    LIB_DIR_NAME: str = "lib"
-    DEFAULT_LIB_DIR: Path = _env_path(
-        "LIB_DIR",
-        user_config_path("abx") / LIB_DIR_NAME,
+    DEFAULT_ABXPKG_LIB_DIR: Path = _env_path(
+        "ABXPKG_LIB_DIR",
+        user_config_path("abx") / "lib",
     )
-    DEFAULT_LIB_BIN_DIR: Path = _env_path("LIB_BIN_DIR", DEFAULT_LIB_DIR / "bin")
 
     RESERVED_ARCHIVE_DIR_NAMES: frozenset[str] = frozenset(
         (
@@ -241,7 +239,7 @@ class ConstantsDict:
             SOURCES_DIR_NAME,
             LOGS_DIR_NAME,
             CACHE_DIR_NAME,
-            LIB_DIR_NAME,
+            "lib",
             TMP_DIR_NAME,
             PERSONAS_DIR_NAME,
             CUSTOM_TEMPLATES_DIR_NAME,

archivebox/config/django.py

@@ -41,6 +41,12 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
         # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
         return
 
+    # SQLite creates index.sqlite3 during django.setup()/migrate. Apply the
+    # ArchiveBox file-mode policy before any DB connection can create the file,
+    # otherwise a permissive parent umask can expose a just-created DB until a
+    # later chmod runs.
+    os.umask(0o777 - (int(CONFIG.OUTPUT_PERMISSIONS, base=8) | 0o111))
+
     # Third-party patches are only needed once Django/apps are about to load.
     # Keeping them out of archivebox.__init__ avoids paying Django/Daphne setup
     # cost for cheap CLI startup paths like `archivebox <cmd> --help`.

archivebox/config/ldap.py

@@ -1,6 +1,6 @@
 __package__ = "archivebox.config"
 
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 
 from archivebox.config.configset import BaseConfigSet
 
@@ -14,6 +14,7 @@ class LDAPConfig(BaseConfigSet):
     """
 
     toml_section_header: str = "LDAP_CONFIG"
+    _scope: str = PrivateAttr(default="server")
 
     LDAP_ENABLED: bool = Field(default=False)
     LDAP_SERVER_URI: str | None = Field(default=None)

archivebox/config/paths.py

@@ -19,7 +19,9 @@ if TYPE_CHECKING:
 #############################################################################################
 
 PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent  # archivebox source code dir
-DATA_DIR: Path = Path(os.environ.get("DATA_DIR", os.getcwd())).resolve()  # archivebox user data dir
+DATA_DIR: Path = Path(os.getcwd()).resolve()  # archivebox user data dir
+MAX_TMP_SOCKET_URL_LENGTH = 90
+SUPERVISORD_SOCKET_FILENAME = "supervisord.sock"
 
 
 def _env_path(key: str, default: Path) -> Path:
@@ -29,8 +31,8 @@ def _env_path(key: str, default: Path) -> Path:
     return path.resolve()
 
 
-ARCHIVE_DIR: Path = _env_path("ARCHIVE_DIR", DATA_DIR / "archive")  # archivebox snapshot data dir
-USERS_DIR: Path = _env_path("USERS_DIR", ARCHIVE_DIR / "users")  # archivebox user-scoped crawl/snapshot data dir
+ARCHIVE_DIR: Path = DATA_DIR / "archive"  # archivebox snapshot data dir
+USERS_DIR: Path = ARCHIVE_DIR / "users"  # archivebox user-scoped crawl/snapshot data dir
 
 IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
 
@@ -120,7 +122,7 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No
     try:
         with SudoPermission(uid=uid, fallback=fallback):
             test_file.exists()
-            test_file.write_text(f"Checking if PUID={uid} PGID={gid} can write to dir")
+            test_file.write_text(f"Checking if uid={uid} gid={gid} can write to dir")
             test_file.unlink()
             return True
     except (OSError, PermissionError):
@@ -176,33 +178,45 @@ def create_and_chown_dir(dir_path: Path) -> None:
 
 
 def tmp_dir_socket_path_is_short_enough(dir_path: Path) -> bool:
-    socket_file = dir_path.absolute().resolve() / "supervisord.sock"
-    return len(f"file://{socket_file}") <= 96
+    socket_file = dir_path.absolute().resolve() / SUPERVISORD_SOCKET_FILENAME
+    return len(f"file://{socket_file}") < MAX_TMP_SOCKET_URL_LENGTH
+
+
+def tmp_dir_candidates(config: "ArchiveBoxConfig") -> list[Path]:
+    from archivebox.config.constants import CONSTANTS
+
+    collection_id = get_collection_id()
+    collection_id_short = collection_id[:4]
+    system_tmp_dir = Path(tempfile.gettempdir())
+    candidates = [
+        config.TMP_DIR,  # <user-specified>
+        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
+        Path("/var/run/archivebox") / collection_id,
+        Path("/tmp") / "archivebox" / collection_id,
+        Path("~/.tmp/archivebox").expanduser() / collection_id,
+        system_tmp_dir / "archivebox" / collection_id,
+        system_tmp_dir / "archivebox" / collection_id_short,
+        system_tmp_dir / "abx" / collection_id_short,
+    ]
+    seen = set()
+    unique_candidates = []
+    for path in candidates:
+        path_key = str(path.expanduser().absolute())
+        if path_key in seen:
+            continue
+        seen.add(path_key)
+        unique_candidates.append(path)
+    return unique_candidates
 
 
 def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
-    from archivebox.config.constants import CONSTANTS
     from archivebox.config.common import get_config
     from archivebox.misc.checks import check_tmp_dir
 
     config = config or get_config(**config_kwargs)
-    # try a few potential directories in order of preference
-    CANDIDATES = [
-        config.TMP_DIR,  # <user-specified>
-        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
-        Path("/var/run/archivebox") / get_collection_id(),  # /var/run/archivebox/abc5d8512
-        Path("/tmp") / "archivebox" / get_collection_id(),  # /tmp/archivebox/abc5d8512
-        Path("~/.tmp/archivebox").expanduser() / get_collection_id(),  # ~/.tmp/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id(),  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d
-        Path(tempfile.gettempdir()) / "abx" / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/abx/abc5
-    ]
+    candidates = tmp_dir_candidates(config)
     fallback_candidate = None
-    for candidate in CANDIDATES:
+    for candidate in candidates:
         try:
             create_and_chown_dir(candidate)
         except Exception:
@@ -231,7 +245,7 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxC
         return fallback_candidate
 
     if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {CANDIDATES}!")
+        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {candidates}!")
 
 
 def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@@ -240,8 +254,8 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
 
     config = config or get_config(**config_kwargs)
 
-    # LIB_DIR is either the shared platformdirs default or an explicit env/config override.
-    CANDIDATES = [config.LIB_DIR]
+    # ABXPKG_LIB_DIR is either the shared platformdirs default or an explicit env/config override.
+    CANDIDATES = [config.ABXPKG_LIB_DIR]
 
     for candidate in CANDIDATES:
         try:
@@ -249,12 +263,12 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
         except Exception:
             pass
         if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True):
-            if autofix and config.LIB_DIR != candidate:
-                os.environ["LIB_DIR"] = str(candidate)
+            if autofix and config.ABXPKG_LIB_DIR != candidate:
+                os.environ["ABXPKG_LIB_DIR"] = str(candidate)
             return candidate
 
     if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable LIB_DIR, tried {CANDIDATES}!")
+        raise OSError(f"ArchiveBox is unable to find a writable ABXPKG_LIB_DIR, tried {CANDIDATES}!")
 
 
 def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@@ -290,20 +304,20 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
             },
             "ARCHIVE_DIR": {
-                "path": config.ARCHIVE_DIR.resolve(),
+                "path": CONSTANTS.ARCHIVE_DIR.resolve(),
                 "enabled": True,
-                "is_valid": os.path.isdir(config.ARCHIVE_DIR)
-                and os.access(config.ARCHIVE_DIR, os.R_OK)
-                and os.access(config.ARCHIVE_DIR, os.W_OK),
-                "is_mount": os.path.ismount(config.ARCHIVE_DIR.resolve()),
+                "is_valid": os.path.isdir(CONSTANTS.ARCHIVE_DIR)
+                and os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK)
+                and os.access(CONSTANTS.ARCHIVE_DIR, os.W_OK),
+                "is_mount": os.path.ismount(CONSTANTS.ARCHIVE_DIR.resolve()),
             },
             "USERS_DIR": {
-                "path": config.USERS_DIR.resolve(),
-                "enabled": os.path.isdir(config.USERS_DIR),
-                "is_valid": os.path.isdir(config.USERS_DIR)
-                and os.access(config.USERS_DIR, os.R_OK)
-                and os.access(config.USERS_DIR, os.W_OK),
-                "is_mount": os.path.ismount(config.USERS_DIR.resolve()),
+                "path": CONSTANTS.USERS_DIR.resolve(),
+                "enabled": os.path.isdir(CONSTANTS.USERS_DIR),
+                "is_valid": os.path.isdir(CONSTANTS.USERS_DIR)
+                and os.access(CONSTANTS.USERS_DIR, os.R_OK)
+                and os.access(CONSTANTS.USERS_DIR, os.W_OK),
+                "is_mount": os.path.ismount(CONSTANTS.USERS_DIR.resolve()),
             },
             "SOURCES_DIR": {
                 "path": CONSTANTS.SOURCES_DIR.resolve(),
@@ -329,7 +343,10 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
             "TMP_DIR": {
                 "path": tmp_dir.resolve(),
                 "enabled": True,
-                "is_valid": os.path.isdir(tmp_dir) and os.access(tmp_dir, os.R_OK) and os.access(tmp_dir, os.W_OK),  # read + write
+                "is_valid": os.path.isdir(tmp_dir)
+                and os.access(tmp_dir, os.R_OK)
+                and os.access(tmp_dir, os.W_OK)
+                and tmp_dir_socket_path_is_short_enough(tmp_dir),
             },
             # "CACHE_DIR": {
             #     "path": CACHE_DIR.resolve(),
@@ -347,11 +364,9 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
 
     config = config or get_config(**config_kwargs)
     try:
-        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR
+        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.ABXPKG_LIB_DIR
     except Exception:
-        lib_dir = config.LIB_DIR
-
-    lib_bin_dir = lib_dir / "bin"
+        lib_dir = config.ABXPKG_LIB_DIR
 
     return AttrDict(
         {
@@ -366,26 +381,19 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 "is_valid": os.access(CONSTANTS.STATIC_DIR, os.R_OK) and os.access(CONSTANTS.STATIC_DIR, os.X_OK),  # read + list
             },
             "CUSTOM_TEMPLATES_DIR": {
-                "path": config.CUSTOM_TEMPLATES_DIR.resolve(),
-                "enabled": os.path.isdir(config.CUSTOM_TEMPLATES_DIR),
-                "is_valid": os.path.isdir(config.CUSTOM_TEMPLATES_DIR) and os.access(config.CUSTOM_TEMPLATES_DIR, os.R_OK),  # read
+                "path": CONSTANTS.CUSTOM_TEMPLATES_DIR.resolve(),
+                "enabled": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR),
+                "is_valid": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK),  # read
             },
             "USER_PLUGINS_DIR": {
                 "path": CONSTANTS.USER_PLUGINS_DIR.resolve(),
                 "enabled": os.path.isdir(CONSTANTS.USER_PLUGINS_DIR),
                 "is_valid": os.path.isdir(CONSTANTS.USER_PLUGINS_DIR) and os.access(CONSTANTS.USER_PLUGINS_DIR, os.R_OK),  # read
             },
-            "LIB_DIR": {
+            "ABXPKG_LIB_DIR": {
                 "path": lib_dir.resolve(),
                 "enabled": True,
                 "is_valid": os.path.isdir(lib_dir) and os.access(lib_dir, os.R_OK) and os.access(lib_dir, os.W_OK),  # read + write
             },
-            "LIB_BIN_DIR": {
-                "path": lib_bin_dir.resolve(),
-                "enabled": True,
-                "is_valid": os.path.isdir(lib_bin_dir)
-                and os.access(lib_bin_dir, os.R_OK)
-                and os.access(lib_bin_dir, os.W_OK),  # read + write
-            },
         },
     )

archivebox/config/permissions.py

@@ -15,8 +15,6 @@ from contextlib import contextmanager
 #############################################################################################
 
 DATA_DIR = Path(os.getcwd())
-if os.environ.get("DATA_DIR") and Path(os.environ["DATA_DIR"]).resolve() != DATA_DIR.resolve():
-    raise SystemExit(f"[X] DATA_DIR={os.environ['DATA_DIR']} must equal cwd={DATA_DIR}; cd into the data dir before running archivebox")
 
 try:
     DATA_DIR_STAT = DATA_DIR.stat()
@@ -26,8 +24,8 @@ except PermissionError:
     DATA_DIR_UID = 0
     DATA_DIR_GID = 0
 
-DEFAULT_PUID = 911
-DEFAULT_PGID = 911
+DEFAULT_UID = 911
+DEFAULT_GID = 911
 RUNNING_AS_UID = os.getuid()
 RUNNING_AS_GID = os.getgid()
 EUID = os.geteuid()
@@ -44,23 +42,20 @@ IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE",
 
 FALLBACK_UID = RUNNING_AS_UID or SUDO_UID
 FALLBACK_GID = RUNNING_AS_GID or SUDO_GID
-if RUNNING_AS_UID == 0:
-    try:
-        # if we are running as root it's really hard to figure out what the correct archivebox user should be
-        # as a last resort instead of setting DATA_DIR ownership to 0:0 (which breaks it for non-root users)
-        # check if 911:911 archivebox user exists on host system, and use it instead of 0
-        if pwd.getpwuid(DEFAULT_PUID).pw_name == "archivebox":
-            FALLBACK_UID = DEFAULT_PUID
-            FALLBACK_GID = DEFAULT_PGID
-    except Exception:
-        pass
+try:
+    ARCHIVEBOX_ACCOUNT = pwd.getpwnam("archivebox")
+except KeyError:
+    ARCHIVEBOX_ACCOUNT = None
 
-
-os.environ.setdefault("PUID", str(DATA_DIR_UID or EUID or RUNNING_AS_UID or FALLBACK_UID))
-os.environ.setdefault("PGID", str(DATA_DIR_GID or EGID or RUNNING_AS_GID or FALLBACK_GID))
-
-ARCHIVEBOX_USER = int(os.environ["PUID"])
-ARCHIVEBOX_GROUP = int(os.environ["PGID"])
+if DATA_DIR_UID != 0:
+    ARCHIVEBOX_USER = DATA_DIR_UID
+    ARCHIVEBOX_GROUP = DATA_DIR_GID
+elif RUNNING_AS_UID == 0 and ARCHIVEBOX_ACCOUNT is not None:
+    ARCHIVEBOX_USER = ARCHIVEBOX_ACCOUNT.pw_uid
+    ARCHIVEBOX_GROUP = ARCHIVEBOX_ACCOUNT.pw_gid
+else:
+    ARCHIVEBOX_USER = EUID or RUNNING_AS_UID or FALLBACK_UID
+    ARCHIVEBOX_GROUP = EGID or RUNNING_AS_GID or FALLBACK_GID
 if not USER:
     try:
         # alternative method 1 to get username
@@ -96,13 +91,12 @@ except Exception:
 
 
 def drop_privileges():
-    """If running as root, drop privileges to the user that owns the data dir (or PUID)"""
+    """If running as root, drop privileges to the data dir owner or archivebox user."""
 
-    # always run archivebox as the user that owns the data dir, never as root
+    # Always run ArchiveBox as the user that owns the data dir, or as the
+    # archivebox service account when the data dir is root-owned.
     if os.getuid() == 0:
-        # drop permissions to the user that owns the data dir / provided PUID
         if os.geteuid() != ARCHIVEBOX_USER and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
-            # drop our effective UID to the archivebox user's UID
             os.seteuid(ARCHIVEBOX_USER)
 
             # update environment variables so that subprocesses dont try to write to /root

archivebox/config/version.py

@@ -6,6 +6,7 @@ import importlib.metadata
 from pathlib import Path
 from functools import cache
 from datetime import datetime
+import re
 
 #############################################################################################
 
@@ -41,11 +42,57 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR):
 
 @cache
 def get_COMMIT_HASH() -> str | None:
+    for env_var in ("ARCHIVEBOX_COMMIT_HASH", "COMMIT_HASH"):
+        env_commit_hash = os.environ.get(env_var, "").strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", env_commit_hash):
+            return env_commit_hash
+
+    if IN_DOCKER:
+        try:
+            version_text = Path("/VERSION.txt").read_text()
+            matches = re.findall(r"^COMMIT_HASH=([0-9a-fA-F]{40})$", version_text, re.MULTILINE)
+            if matches:
+                return matches[-1]
+        except Exception:
+            pass
+
+    def _read_git_file(git_dir: Path, ref: str) -> str | None:
+        try:
+            return git_dir.joinpath(ref).read_text().strip()
+        except Exception:
+            pass
+
+        try:
+            packed_refs = git_dir.joinpath("packed-refs").read_text().splitlines()
+        except Exception:
+            return None
+
+        for line in packed_refs:
+            if line.startswith("#") or line.startswith("^") or not line.strip():
+                continue
+            commit_hash, packed_ref = line.split(" ", 1)
+            if packed_ref == ref:
+                return commit_hash.strip()
+
+        return None
+
     try:
         git_dir = PACKAGE_DIR.parent / ".git"
-        ref = (git_dir / "HEAD").read_text().strip().split(" ")[-1]
-        commit_hash = git_dir.joinpath(ref).read_text().strip()
-        return commit_hash
+        if git_dir.is_file():
+            gitdir_line = git_dir.read_text().strip()
+            gitdir_path = gitdir_line.removeprefix("gitdir:").strip()
+            git_dir = Path(gitdir_path)
+            if not git_dir.is_absolute():
+                git_dir = PACKAGE_DIR.parent / git_dir
+
+        head = (git_dir / "HEAD").read_text().strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", head):
+            return head
+
+        ref = head.removeprefix("ref:").strip()
+        commit_hash = _read_git_file(git_dir, ref)
+        if commit_hash:
+            return commit_hash
     except Exception:
         pass
 
@@ -59,13 +106,10 @@ def get_COMMIT_HASH() -> str | None:
 
 @cache
 def get_BUILD_TIME() -> str:
-    if IN_DOCKER:
-        try:
-            # if we're in the archivebox official docker image, /VERSION.txt will contain the build time
-            docker_build_end_time = Path("/VERSION.txt").read_text().rsplit("BUILD_END_TIME=")[-1].split("\n", 1)[0]
-            return docker_build_end_time
-        except Exception:
-            pass
+    for env_var in ("ARCHIVEBOX_BUILD_TIME", "BUILD_TIME"):
+        build_time = os.environ.get(env_var, "").strip()
+        if build_time:
+            return build_time
 
     src_last_modified_unix_timestamp = (PACKAGE_DIR / "README.md").stat().st_mtime
     return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime("%Y-%m-%d %H:%M:%S %s")

archivebox/config/views.py

@@ -1,13 +1,9 @@
 __package__ = "archivebox.config"
 
-import html
-import json
 import os
 import inspect
-import re
 from pathlib import Path
 from typing import Any
-from collections.abc import Callable
 from urllib.parse import quote, urlencode
 from django.http import HttpRequest
 from django.utils import timezone
@@ -22,15 +18,12 @@ from archivebox.misc.util import parse_date
 
 from archivebox.machine.models import Binary
 
-ABX_PLUGINS_DOCS_BASE_URL = "https://archivebox.github.io/abx-plugins/"
-ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/"
-LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/"
 INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"
 
 
 def is_superuser(request: HttpRequest) -> bool:
-    return bool(getattr(request.user, "is_superuser", False))
+    return bool(request.user.is_superuser)
 
 
 def format_parsed_datetime(value: object) -> str:
@@ -38,59 +31,6 @@ def format_parsed_datetime(value: object) -> str:
     return parsed.strftime("%Y-%m-%d %H:%M:%S") if parsed else ""
 
 
-JSON_TOKEN_RE = re.compile(
-    r'(?P<key>"(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*")(?=\s*:)'
-    r'|(?P<string>"(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*")'
-    r"|(?P<boolean>\btrue\b|\bfalse\b)"
-    r"|(?P<null>\bnull\b)"
-    r"|(?P<number>-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)",
-)
-
-
-def render_code_block(text: str, *, highlighted: bool = False) -> str:
-    code = html.escape(text, quote=False)
-
-    if highlighted:
-
-        def _wrap_token(match: re.Match[str]) -> str:
-            styles = {
-                "key": "color: #0550ae;",
-                "string": "color: #0a7f45;",
-                "boolean": "color: #8250df; font-weight: 600;",
-                "null": "color: #6e7781; font-style: italic;",
-                "number": "color: #b35900;",
-            }
-            token_type = next(name for name, value in match.groupdict().items() if value is not None)
-            return f'<span style="{styles[token_type]}">{match.group(0)}</span>'
-
-        code = JSON_TOKEN_RE.sub(_wrap_token, code)
-
-    return (
-        '<pre style="max-height: 600px; overflow: auto; background: #f6f8fa; '
-        'border: 1px solid #d0d7de; border-radius: 6px; padding: 12px; margin: 0;">'
-        '<code style="font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, '
-        "'Liberation Mono', monospace; white-space: pre; line-height: 1.5;\">"
-        f"{code}"
-        "</code></pre>"
-    )
-
-
-def render_highlighted_json_block(value: Any) -> str:
-    return render_code_block(json.dumps(value, indent=2, ensure_ascii=False), highlighted=True)
-
-
-def get_plugin_docs_url(plugin_name: str) -> str:
-    return f"{ABX_PLUGINS_DOCS_BASE_URL}#{plugin_name}"
-
-
-def get_plugin_hook_source_url(plugin_name: str, hook_name: str) -> str:
-    return f"{ABX_PLUGINS_GITHUB_BASE_URL}{quote(plugin_name)}/{quote(hook_name)}"
-
-
-def get_live_config_url(key: str) -> str:
-    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
-
-
 def get_environment_binary_url(name: str) -> str:
     return f"{ENVIRONMENT_BINARIES_BASE_URL}{quote(name)}/"
 
@@ -99,200 +39,11 @@ def get_installed_binary_change_url(name: str, binary: Binary | None) -> str | N
     if binary is None or not binary.id:
         return None
 
-    base_url = getattr(binary, "admin_change_url", None) or f"{INSTALLED_BINARIES_BASE_URL}{binary.id}/change/"
+    base_url = binary.admin_change_url
     changelist_filters = urlencode({"q": name})
     return f"{base_url}?{urlencode({'_changelist_filters': changelist_filters})}"
 
 
-def get_machine_admin_url() -> str | None:
-    try:
-        from archivebox.machine.models import Machine
-
-        machine = Machine.current()
-        return getattr(machine, "admin_change_url", None) or f"/admin/machine/machine/{machine.id.hex}/change/"
-    except Exception:
-        return None
-
-
-def render_code_tag_list(values: list[str]) -> str:
-    if not values:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    tags = "".join(
-        str(
-            format_html(
-                '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>',
-                value,
-            ),
-        )
-        for value in values
-    )
-    return f'<div style="display: flex; flex-wrap: wrap;">{tags}</div>'
-
-
-def render_plugin_metadata_html(config: dict[str, Any]) -> str:
-    required_binaries = [
-        str(item.get("name")) for item in (config.get("required_binaries") or []) if isinstance(item, dict) and item.get("name")
-    ]
-    rows = (
-        ("Title", config.get("title") or "(none)"),
-        ("Description", config.get("description") or "(none)"),
-        ("Required Plugins", mark_safe(render_link_tag_list(config.get("required_plugins") or [], get_plugin_docs_url))),
-        ("Required Binaries", mark_safe(render_link_tag_list(required_binaries, get_environment_binary_url))),
-        ("Output MIME Types", mark_safe(render_code_tag_list(config.get("output_mimetypes") or []))),
-    )
-
-    rendered_rows = "".join(
-        str(
-            format_html(
-                '<div style="margin: 0 0 14px 0;"><div style="font-weight: 600; margin-bottom: 4px;">{}</div><div>{}</div></div>',
-                label,
-                value,
-            ),
-        )
-        for label, value in rows
-    )
-    return f'<div style="margin: 4px 0 0 0;">{rendered_rows}</div>'
-
-
-def render_link_tag_list(values: list[str], url_resolver: Callable[[str], str] | None = None) -> str:
-    if not values:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    tags = []
-    for value in values:
-        if url_resolver is None:
-            tags.append(
-                str(
-                    format_html(
-                        '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                        'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>',
-                        value,
-                    ),
-                ),
-            )
-        else:
-            tags.append(
-                str(
-                    format_html(
-                        '<a href="{}" style="text-decoration: none;">'
-                        '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                        'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>'
-                        "</a>",
-                        url_resolver(value),
-                        value,
-                    ),
-                ),
-            )
-    return f'<div style="display: flex; flex-wrap: wrap;">{"".join(tags)}</div>'
-
-
-def render_property_links(prop_name: str, prop_info: dict[str, Any], machine_admin_url: str | None) -> str:
-    links = [
-        str(format_html('<a href="{}">Computed value</a>', get_live_config_url(prop_name))),
-    ]
-    if machine_admin_url:
-        links.append(str(format_html('<a href="{}">Edit override</a>', machine_admin_url)))
-
-    fallback = prop_info.get("x-fallback")
-    if isinstance(fallback, str) and fallback:
-        links.append(str(format_html('<a href="{}">Fallback: <code>{}</code></a>', get_live_config_url(fallback), fallback)))
-
-    aliases = prop_info.get("x-aliases") or []
-    if isinstance(aliases, list):
-        for alias in aliases:
-            if isinstance(alias, str) and alias:
-                links.append(str(format_html('<a href="{}">Alias: <code>{}</code></a>', get_live_config_url(alias), alias)))
-
-    default = prop_info.get("default")
-    if prop_name.endswith("_BINARY") and isinstance(default, str) and default:
-        links.append(str(format_html('<a href="{}">Binary: <code>{}</code></a>', get_environment_binary_url(default), default)))
-
-    return " &nbsp; ".join(links)
-
-
-def render_config_properties_html(properties: dict[str, Any], machine_admin_url: str | None) -> str:
-    header_links = [
-        str(format_html('<a href="{}">Dependencies</a>', ENVIRONMENT_BINARIES_BASE_URL)),
-        str(format_html('<a href="{}">Installed Binaries</a>', INSTALLED_BINARIES_BASE_URL)),
-    ]
-    if machine_admin_url:
-        header_links.insert(0, str(format_html('<a href="{}">Machine Config Editor</a>', machine_admin_url)))
-
-    cards = [
-        f'<div style="margin: 0 0 16px 0;">{" &nbsp; | &nbsp; ".join(header_links)}</div>',
-    ]
-
-    for prop_name, prop_info in properties.items():
-        prop_type = prop_info.get("type", "unknown")
-        if isinstance(prop_type, list):
-            prop_type = " | ".join(str(type_name) for type_name in prop_type)
-        prop_desc = prop_info.get("description", "")
-
-        default_html = ""
-        if "default" in prop_info:
-            default_html = str(
-                format_html(
-                    '<div style="margin-top: 6px;"><b>Default:</b> <code>{}</code></div>',
-                    prop_info["default"],
-                ),
-            )
-
-        description_html = prop_desc or mark_safe('<span style="color: #6e7781;">(no description)</span>')
-        cards.append(
-            str(
-                format_html(
-                    '<div style="margin: 0 0 14px 0; padding: 12px; background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 6px;">'
-                    '<div style="margin-bottom: 6px;">'
-                    '<a href="{}" style="font-weight: 600;"><code>{}</code></a>'
-                    ' <span style="color: #6e7781;">({})</span>'
-                    "</div>"
-                    '<div style="margin-bottom: 6px;">{}</div>'
-                    '<div style="font-size: 0.95em;">{}</div>'
-                    "{}"
-                    "</div>",
-                    get_live_config_url(prop_name),
-                    prop_name,
-                    prop_type,
-                    description_html,
-                    mark_safe(render_property_links(prop_name, prop_info, machine_admin_url)),
-                    mark_safe(default_html),
-                ),
-            ),
-        )
-
-    return "".join(cards)
-
-
-def render_hook_links_html(plugin_name: str, hooks: list[str], source: str) -> str:
-    if not hooks:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    items = []
-    for hook_name in hooks:
-        if source == "builtin":
-            items.append(
-                str(
-                    format_html(
-                        '<div style="margin: 0 0 8px 0;"><a href="{}" target="_blank" rel="noopener noreferrer"><code>{}</code></a></div>',
-                        get_plugin_hook_source_url(plugin_name, hook_name),
-                        hook_name,
-                    ),
-                ),
-            )
-        else:
-            items.append(
-                str(
-                    format_html(
-                        '<div style="margin: 0 0 8px 0;"><code>{}</code></div>',
-                        hook_name,
-                    ),
-                ),
-            )
-    return "".join(items)
-
-
 def render_binary_detail_description(name: str, merged: dict[str, Any], db_binary: Any) -> str:
     installed_binary_url = get_installed_binary_change_url(name, db_binary)
 
@@ -386,48 +137,6 @@ def get_db_binaries_by_name() -> dict[str, Binary]:
     return {name: max(records, key=_binary_sort_key) for name, records in grouped.items()}
 
 
-def get_filesystem_plugins() -> dict[str, dict[str, Any]]:
-    """Discover plugins from filesystem directories."""
-    import json
-    from archivebox.hooks import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR
-
-    plugins = {}
-
-    for base_dir, source in [(BUILTIN_PLUGINS_DIR, "builtin"), (USER_PLUGINS_DIR, "user")]:
-        if not base_dir.exists():
-            continue
-
-        for plugin_dir in base_dir.iterdir():
-            if plugin_dir.is_dir() and not plugin_dir.name.startswith("_"):
-                plugin_id = f"{source}.{plugin_dir.name}"
-
-                # Find hook scripts
-                hooks = []
-                for ext in ("sh", "py", "js"):
-                    hooks.extend(plugin_dir.glob(f"on_*__*.{ext}"))
-
-                # Load config.json if it exists
-                config_file = plugin_dir / "config.json"
-                config_data = None
-                if config_file.exists():
-                    try:
-                        with open(config_file) as f:
-                            config_data = json.load(f)
-                    except (json.JSONDecodeError, OSError):
-                        config_data = None
-
-                plugins[plugin_id] = {
-                    "id": plugin_id,
-                    "name": plugin_dir.name,
-                    "path": str(plugin_dir),
-                    "source": source,
-                    "hooks": [str(h.name) for h in hooks],
-                    "config": config_data,
-                }
-
-    return plugins
-
-
 @render_with_table_view
 def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."
@@ -444,7 +153,7 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and getattr(binary, "is_valid", getattr(binary, "abspath", None)))
+        binary_is_valid = bool(binary and binary.is_valid)
 
         rows["Binary Name"].append(ItemLink(name, key=name))
 
@@ -471,9 +180,9 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
         "youtube-dl": "yt-dlp",
     }.get(key, key)
     db_binary = get_db_binaries_by_name().get(key)
-    binary_is_valid = bool(db_binary and getattr(db_binary, "is_valid", getattr(db_binary, "abspath", None)))
+    binary_is_valid = bool(db_binary and db_binary.is_valid)
     if binary_is_valid:
-        binary_data = db_binary.to_json() if hasattr(db_binary, "to_json") else db_binary.__dict__
+        binary_data = db_binary.to_json()
         section: SectionData = {
             "name": key,
             "description": mark_safe(render_binary_detail_description(key, binary_data, db_binary)),
@@ -512,131 +221,6 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     )
 
 
-@render_with_table_view
-def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    assert is_superuser(request), "Must be a superuser to view configuration settings."
-
-    rows = {
-        "Name": [],
-        "Source": [],
-        "Path": [],
-        "Hooks": [],
-        "Config": [],
-    }
-
-    plugins = get_filesystem_plugins()
-
-    for plugin_id, plugin in plugins.items():
-        rows["Name"].append(ItemLink(plugin["name"], key=plugin_id))
-        rows["Source"].append(plugin["source"])
-        rows["Path"].append(format_html("<code>{}</code>", plugin["path"]))
-        rows["Hooks"].append(", ".join(plugin["hooks"]) or "(none)")
-
-        # Show config status
-        if plugin.get("config"):
-            config_properties = plugin["config"].get("properties", {})
-            config_count = len(config_properties)
-            rows["Config"].append(f"✅ {config_count} properties" if config_count > 0 else "✅ present")
-        else:
-            rows["Config"].append("❌ none")
-
-    if not plugins:
-        # Show a helpful message when no plugins found
-        rows["Name"].append("(no plugins found)")
-        rows["Source"].append("-")
-        rows["Path"].append(mark_safe("<code>abx_plugins/plugins/</code> or <code>data/custom_plugins/</code>"))
-        rows["Hooks"].append("-")
-        rows["Config"].append("-")
-
-    return TableContext(
-        title="Installed plugins",
-        table=rows,
-    )
-
-
-@render_with_item_view
-def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-    assert is_superuser(request), "Must be a superuser to view configuration settings."
-
-    plugins = get_filesystem_plugins()
-
-    plugin = plugins.get(key)
-    if not plugin:
-        return ItemContext(
-            slug=key,
-            title=f"Plugin not found: {key}",
-            data=[],
-        )
-
-    # Base fields that all plugins have
-    docs_url = get_plugin_docs_url(plugin["name"])
-    machine_admin_url = get_machine_admin_url()
-    fields = {
-        "id": plugin["id"],
-        "name": plugin["name"],
-        "source": plugin["source"],
-    }
-
-    sections: list[SectionData] = [
-        {
-            "name": plugin["name"],
-            "description": format_html(
-                '<code>{}</code><br/><a href="{}" target="_blank" rel="noopener noreferrer">ABX Plugin Docs</a>',
-                plugin["path"],
-                docs_url,
-            ),
-            "fields": fields,
-            "help_texts": {},
-        },
-    ]
-
-    if plugin["hooks"]:
-        sections.append(
-            {
-                "name": "Hooks",
-                "description": mark_safe(render_hook_links_html(plugin["name"], plugin["hooks"], plugin["source"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-    if plugin.get("config"):
-        sections.append(
-            {
-                "name": "Plugin Metadata",
-                "description": mark_safe(render_plugin_metadata_html(plugin["config"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-        sections.append(
-            {
-                "name": "config.json",
-                "description": mark_safe(render_highlighted_json_block(plugin["config"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-        config_properties = plugin["config"].get("properties", {})
-        if config_properties:
-            sections.append(
-                {
-                    "name": "Config Properties",
-                    "description": mark_safe(render_config_properties_html(config_properties, machine_admin_url)),
-                    "fields": {},
-                    "help_texts": {},
-                },
-            )
-
-    return ItemContext(
-        slug=key,
-        title=plugin["name"],
-        data=sections,
-    )
-
-
 @render_with_table_view
 def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."

archivebox/core/admin_archiveresults.py

@@ -19,14 +19,12 @@ from django.urls import reverse, resolve
 from django.utils import timezone
 from django.utils.text import smart_split
 
-from archivebox.config import DATA_DIR
-from archivebox.config.common import get_config
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.base_models.admin import BaseModelAdmin
-from archivebox.hooks import get_plugin_icon
-from archivebox.core.host_util import build_snapshot_url
+from archivebox.plugins.discovery import get_plugin_icon
+from archivebox.plugins.views import LIVE_PLUGIN_BASE_URL
+from archivebox.core.routes_util import build_snapshot_url
 from archivebox.core.widgets import InlineTagEditorWidget
-from archivebox.core.views import LIVE_PLUGIN_BASE_URL
 from archivebox.machine.env_util import env_to_shell_exports
 
 
@@ -35,7 +33,7 @@ from archivebox.core.models import ArchiveResult, Snapshot
 
 def _get_replay_source_url(result: ArchiveResult) -> str:
     process = result.process_record
-    return str(getattr(process, "url", None) or result.snapshot.url or "")
+    return str((process.url if process else None) or result.snapshot.url or "")
 
 
 def build_abx_dl_display_command(result: ArchiveResult) -> str:
@@ -51,8 +49,8 @@ def build_abx_dl_display_command(result: ArchiveResult) -> str:
 
 def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str:
     display_command = build_abx_dl_display_command(result)
-    process = getattr(result, "process", None)
-    env_items = env_to_shell_exports(getattr(process, "env", None) or {})
+    process = result.process
+    env_items = env_to_shell_exports(process.env if process else {})
     if config is not None:
         result.snapshot._runtime_config = config
     snapshot_dir = shlex.quote(str(result.pwd or result.snapshot_dir))
@@ -62,7 +60,7 @@ def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str:
 
 
 def get_plugin_admin_url(plugin_name: str) -> str:
-    from archivebox.hooks import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, iter_plugin_dirs
+    from archivebox.plugins.discovery import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, iter_plugin_dirs
 
     plugin_dir = next((path.resolve() for path in iter_plugin_dirs() if path.name == plugin_name), None)
     if plugin_dir:
@@ -77,6 +75,12 @@ def get_plugin_admin_url(plugin_name: str) -> str:
     return f"{LIVE_PLUGIN_BASE_URL}builtin.{quote(plugin_name)}/"
 
 
+def get_process_link_label(process) -> str:
+    if process.pid:
+        return str(process.pid)
+    return str(process.id)[-8:]
+
+
 def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
     """Render a nice inline list view of archive results with status, plugin, output, and actions."""
 
@@ -138,25 +142,31 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         process = result.process_record
         process_display = "-"
         if process:
+            process_url = html.escape(reverse("admin:machine_process_change", args=[process.id]), quote=True)
+            process_label = html.escape(get_process_link_label(process), quote=True)
             process_display = f'''
-                <a href="{reverse("admin:machine_process_change", args=[process.id])}"
+                <a href="{process_url}"
                    style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
-                   title="View process">{process.pid or "-"}</a>
+                   title="View process">{process_label}</a>
             '''
 
         machine_display = "-"
         if process and process.machine_id:
+            machine_url = html.escape(reverse("admin:machine_machine_change", args=[process.machine_id]), quote=True)
+            machine_label = html.escape(str(process.machine.hostname or ""), quote=True)
             machine_display = f'''
-                <a href="{reverse("admin:machine_machine_change", args=[process.machine_id])}"
+                <a href="{machine_url}"
                    style="color: #2563eb; text-decoration: none; font-size: 12px;"
-                   title="View machine">{process.machine.hostname}</a>
+                   title="View machine">{machine_label}</a>
             '''
 
         # Truncate output for display
-        full_output = result.output_str_for_display() or "-"
-        output_display = full_output[:60]
-        if len(full_output) > 60:
-            output_display += "..."
+        full_output_raw = result.output_str_for_display() or "-"
+        output_display_raw = full_output_raw[:60]
+        if len(full_output_raw) > 60:
+            output_display_raw += "..."
+        full_output = html.escape(full_output_raw)
+        output_display = html.escape(output_display_raw)
 
         display_cmd = build_abx_dl_display_command(result)
         replay_cmd = build_abx_dl_replay_command(result, config=config)
@@ -164,15 +174,19 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         cmd_attr = html.escape(replay_cmd, quote=True)
 
         # Build output link - use embed_path() which checks output_files first
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else None
-        snapshot_id = str(getattr(result, "snapshot_id", ""))
+        embed_path = result.embed_path()
+        snapshot_id = str(result.snapshot_id)
         if embed_path and result.status == "succeeded":
             output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
         else:
             output_link = build_snapshot_url(snapshot_id, "", config=config)
+        output_link_attr = html.escape(output_link, quote=True)
 
         # Get version - try cmd_version field
-        version = result.cmd_version if result.cmd_version else "-"
+        version = html.escape(str(result.cmd_version if result.cmd_version else "-"), quote=True)
+        plugin_text = html.escape(str(result.plugin or ""), quote=True)
+        status_text = html.escape(str(status), quote=True)
+        pwd_text = html.escape(str(result.pwd or "-"), quote=True)
 
         # Unique ID for this row's expandable output
         row_id = f"output_{idx}_{str(result.id)[:8]}"
@@ -189,18 +203,18 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                 <td style="padding: 10px 12px; white-space: nowrap;">
                     <span style="display: inline-block; padding: 3px 10px; border-radius: 12px;
                                  font-size: 11px; font-weight: 600; text-transform: uppercase;
-                                 color: {color}; background: {bg};">{status}</span>
+                                 color: {color}; background: {bg};">{status_text}</span>
                 </td>
-                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{result.plugin}">
+                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{plugin_text}">
                     {icon}
                 </td>
                 <td style="padding: 10px 12px; font-weight: 500; color: #334155;">
-                        <a href="{output_link}" target="_blank"
+                        <a href="{output_link_attr}" target="_blank"
                            style="color: #334155; text-decoration: none;"
                        title="View output fullscreen"
                        onmouseover="this.style.color='#2563eb'; this.style.textDecoration='underline';"
                        onmouseout="this.style.color='#334155'; this.style.textDecoration='none';">
-                        {result.plugin}
+                        {plugin_text}
                     </a>
                 </td>
                 <td style="padding: 10px 12px; max-width: 280px;">
@@ -227,7 +241,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                 </td>
                 <td style="padding: 10px 8px; white-space: nowrap;">
                     <div style="display: flex; gap: 4px;">
-                        <a href="{output_link}" target="_blank"
+                        <a href="{output_link_attr}" target="_blank"
                            style="padding: 4px 8px; background: #f1f5f9; border-radius: 4px; color: #475569; text-decoration: none; font-size: 11px;"
                            title="View output">📄</a>
                         <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
@@ -246,7 +260,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                             <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
                                 <span style="margin-right: 16px;"><b>ID:</b> <code>{str(result.id)}</code></span>
                                 <span style="margin-right: 16px;"><b>Version:</b> <code>{version}</code></span>
-                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{result.pwd or "-"}</code></span>
+                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{pwd_text}</code></span>
                             </div>
                             <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
                                 <b>Output:</b>
@@ -354,8 +368,7 @@ class ArchiveResultInline(admin.TabularInline):
     def get_formset(self, request, obj=None, **kwargs):
         formset = super().get_formset(request, obj, **kwargs)
         snapshot = self.get_parent_object_from_request(request)
-        form_class = getattr(formset, "form", None)
-        base_fields = getattr(form_class, "base_fields", {})
+        base_fields = formset.form.base_fields
         snapshot_output_dir = str(snapshot.output_dir) if snapshot else ""
 
         # import ipdb; ipdb.set_trace()
@@ -491,12 +504,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     def change_view(self, request, object_id, form_url="", extra_context=None):
         self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         return super().change_view(request, object_id, form_url, extra_context)
 
     def changelist_view(self, request, extra_context=None):
         self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         saved_list_per_page = self.list_per_page
         self.list_per_page = request.archivebox_config.SNAPSHOTS_PER_PAGE
         try:
@@ -522,7 +533,6 @@ class ArchiveResultAdmin(BaseModelAdmin):
             super()
             .get_queryset(request)
             .defer(
-                "config",
                 "notes",
                 "output_json",
             )
@@ -573,20 +583,20 @@ class ArchiveResultAdmin(BaseModelAdmin):
         return queryset.filter(reduce(and_, filters)).distinct(), True
 
     def get_snapshot_view_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        return build_snapshot_url(str(result.snapshot_id), request=request, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return build_snapshot_url(str(result.snapshot_id), request=request, config=request.archivebox_config)
 
     def get_output_view_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
-        output_path = result.embed_path() if hasattr(result, "embed_path") else None
+        request = self.request
+        config = request.archivebox_config
+        output_path = result.embed_path()
         if not output_path:
             output_path = result.plugin or ""
         return build_snapshot_url(str(result.snapshot_id), output_path, request=request, config=config)
 
     def get_output_files_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
+        request = self.request
+        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=request.archivebox_config)}/?files=1"
 
     def get_output_zip_url(self, result: ArchiveResult) -> str:
         return f"{self.get_output_files_url(result)}&download=zip"
@@ -612,10 +622,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
     )
     def snapshot_info(self, result):
         snapshot_id = str(result.snapshot_id)
-        request = getattr(self, "request", None)
+        request = self.request
         return format_html(
             '<a href="{}"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
             snapshot_id[:8],
             result.snapshot.bookmarked_at.strftime("%Y-%m-%d %H:%M"),
             result.snapshot.url[:128],
@@ -665,7 +675,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
         process = result.process_record
         if not process:
             return "-"
-        process_label = process.pid or "-"
+        process_label = get_process_link_label(process)
         return format_html(
             '<a href="{}"><code>{}</code></a>',
             reverse("admin:machine_process_change", args=[process.id]),
@@ -679,17 +689,16 @@ class ArchiveResultAdmin(BaseModelAdmin):
             return "-"
         machine = process.machine
         return format_html(
-            '<a href="{}"><code>{}</code> {}</a>',
+            '<a href="{}">{}</a>',
             reverse("admin:machine_machine_change", args=[machine.id]),
-            str(machine.id)[:8],
             machine.hostname,
         )
 
     @admin.display(description="Command")
     def cmd_str(self, result):
-        request = getattr(self, "request", None)
+        request = self.request
         display_cmd = build_abx_dl_display_command(result)
-        replay_cmd = build_abx_dl_replay_command(result, config=getattr(request, "archivebox_config", None))
+        replay_cmd = build_abx_dl_replay_command(result, config=request.archivebox_config)
         return format_html(
             """
             <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
@@ -710,10 +719,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
         )
 
     def output_display(self, result):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         # Determine output link path - use embed_path() which checks output_files
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else None
+        embed_path = result.embed_path()
         output_path = embed_path if (result.status == "succeeded" and embed_path) else "index.html"
         snapshot_id = str(result.snapshot_id)
         return format_html(
@@ -728,12 +737,12 @@ class ArchiveResultAdmin(BaseModelAdmin):
         if not output_text:
             return "-"
 
-        request = getattr(self, "request", None)
-        live_path = result.embed_path() if hasattr(result, "embed_path") else None
+        request = self.request
+        live_path = result.embed_path()
         if live_path:
             return format_html(
                 '<a href="{}" title="{}"><code>{}</code></a>',
-                build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=getattr(request, "archivebox_config", None)),
+                build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=request.archivebox_config),
                 output_text,
                 output_text,
             )
@@ -785,18 +794,18 @@ class ArchiveResultAdmin(BaseModelAdmin):
         )
 
     def output_summary(self, result):
-        snapshot_dir = Path(DATA_DIR) / str(result.pwd).split("data/", 1)[-1]
+        snapshot_dir = Path(result.snapshot.output_dir)
         output_html = format_html(
             '<pre style="display: inline-block">{}</pre><br/>',
             result.output_str_for_display(),
         )
         snapshot_id = str(result.snapshot_id)
-        request = getattr(self, "request", None)
+        request = self.request
         output_html += format_html(
             '<a href="{}#all">See result files ...</a><br/><pre><code>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
         )
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else ""
+        embed_path = result.embed_path() or ""
         path_from_embed = snapshot_dir / (embed_path or "")
         output_html += format_html(
             '<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>',

archivebox/core/admin_site.py

@@ -3,7 +3,15 @@ __package__ = "archivebox.core"
 from typing import TYPE_CHECKING, Any
 
 from django.contrib import admin
+from django.contrib.auth import REDIRECT_FIELD_NAME
+from django.contrib.auth.decorators import login_not_required
+from django.contrib.auth.views import LoginView
 from django.db import DatabaseError, connection
+from django.http import HttpResponseRedirect
+from django.urls import reverse
+from django.utils.decorators import method_decorator
+from django.utils.translation import gettext as _
+from django.views.decorators.cache import never_cache
 from admin_data_views.admin import (
     admin_data_index_view as adv_admin_data_index_view,
     get_admin_data_urls as adv_get_admin_data_urls,
@@ -12,6 +20,7 @@ from admin_data_views.admin import (
 
 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH
+from archivebox.core.routes_util import is_allowed_archivebox_redirect_url
 
 if TYPE_CHECKING:
     from django.http import HttpRequest
@@ -21,6 +30,17 @@ if TYPE_CHECKING:
     from admin_data_views.typing import AppDict
 
 
+class ArchiveBoxLoginView(LoginView):
+    def get_redirect_url(self) -> str:
+        redirect_to = self.request.POST.get(
+            self.redirect_field_name,
+            self.request.GET.get(self.redirect_field_name),
+        )
+        if is_allowed_archivebox_redirect_url(redirect_to, request=self.request):
+            return redirect_to
+        return ""
+
+
 class ArchiveBoxAdmin(admin.AdminSite):
     site_header = "ArchiveBox"
     index_title = "Admin Views"
@@ -72,6 +92,34 @@ class ArchiveBoxAdmin(admin.AdminSite):
     def admin_data_index_view(self, request: "HttpRequest", **kwargs: Any) -> "TemplateResponse":
         return adv_admin_data_index_view(self, request, **kwargs)
 
+    @method_decorator(never_cache)
+    @login_not_required
+    def login(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
+        if request.method == "GET" and self.has_permission(request):
+            return HttpResponseRedirect(reverse("admin:index", current_app=self.name))
+
+        from django.contrib.admin.forms import AdminAuthenticationForm
+
+        context = {
+            **self.each_context(request),
+            "title": _("Log in"),
+            "subtitle": None,
+            "app_path": request.get_full_path(),
+            "username": request.user.get_username(),
+        }
+        if REDIRECT_FIELD_NAME not in request.GET and REDIRECT_FIELD_NAME not in request.POST:
+            context[REDIRECT_FIELD_NAME] = reverse("admin:index", current_app=self.name)
+        context.update(extra_context or {})
+
+        index_path = reverse("admin:index", current_app=self.name)
+        request.current_app = self.name
+        return ArchiveBoxLoginView.as_view(
+            extra_context=context,
+            authentication_form=self.login_form or AdminAuthenticationForm,
+            template_name=self.login_template or "admin/login.html",
+            next_page=index_path,
+        )(request)
+
     def index(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
         response = super().index(request, extra_context)
         if connection.vendor != "sqlite":

archivebox/core/admin_snapshots.py

@@ -1,20 +1,13 @@
 __package__ = "archivebox.core"
 
-import asyncio
 import json
-import threading
-from copy import copy
 from functools import lru_cache
-from queue import Full, Queue
 from types import SimpleNamespace
-from urllib.parse import urlsplit
-from uuid import UUID
 
 from django.contrib import admin, messages
 from django.urls import path, reverse
 from django.shortcuts import get_object_or_404, redirect
-from django.core.cache import cache
-from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed, QueryDict, StreamingHttpResponse
+from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed
 from django.utils import timezone
 from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
@@ -27,15 +20,20 @@ from archivebox.config.common import get_config
 from archivebox.misc.util import htmldecode, urldecode
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.misc.logging_util import printable_filesize
-from archivebox.search.admin import SEARCH_RESULT_CACHE_TTL, SearchResultsAdminMixin, SearchResultsChangeList, get_admin_search_cache_key
-from archivebox.core.host_util import build_snapshot_url, build_web_url
+from archivebox.search.admin import SearchResultsAdminMixin, SearchResultsChangeList
+from archivebox.search.views import admin_snapshot_search_stream_view
+from archivebox.core.routes_util import build_snapshot_url, build_web_url
 from archivebox.core.tag_util import get_or_create_tag
-from archivebox.hooks import discover_hooks, get_plugin_icon, get_plugin_name, get_plugins
+from archivebox.plugins.hooks import discover_hooks
+from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_plugins
 
 from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 
 from archivebox.core.models import Tag, Snapshot, ArchiveResult
+from archivebox.crawls.models import Crawl
 from archivebox.core.admin_archiveresults import render_archiveresults_list
+from archivebox.core.preview_util import EXTENSION_SCREENSHOT_PLUGIN
+from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
     PERMISSIONS_META,
@@ -43,7 +41,6 @@ from archivebox.core.permissions import (
     normalize_permissions,
 )
 from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget
-from archivebox.crawls.models import Crawl
 
 
 # GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False}
@@ -112,9 +109,7 @@ class SnapshotPermissionsListFilter(admin.SimpleListFilter):
     def queryset(self, request, queryset):
         value = self.value()
         if value:
-            return queryset.filter(
-                Q(permissions=value) | (Q(permissions__isnull=True) & Q(crawl__permissions=value)),
-            )
+            return queryset.filter(permissions=value)
         return queryset
 
 
@@ -132,56 +127,6 @@ class SnapshotStatusListFilter(admin.SimpleListFilter):
         return queryset
 
 
-class SnapshotDepthListFilter(admin.SimpleListFilter):
-    title = "depth"
-    parameter_name = "depth_bucket"
-
-    def lookups(self, request, model_admin):
-        return (
-            ("0", "0 root"),
-            ("1", "1"),
-            ("2", "2"),
-            ("3plus", "3+"),
-        )
-
-    def queryset(self, request, queryset):
-        value = self.value()
-        if value == "0":
-            return queryset.filter(depth=0)
-        if value == "1":
-            return queryset.filter(depth=1)
-        if value == "2":
-            return queryset.filter(depth=2)
-        if value == "3plus":
-            return queryset.filter(depth__gte=3)
-        return queryset
-
-
-class SnapshotRelationListFilter(admin.SimpleListFilter):
-    title = "crawl position"
-    parameter_name = "position"
-
-    def lookups(self, request, model_admin):
-        return (
-            ("root", "Root URL"),
-            ("discovered", "Discovered URL"),
-            ("has_children", "Has discovered URLs"),
-            ("no_children", "No discovered URLs"),
-        )
-
-    def queryset(self, request, queryset):
-        value = self.value()
-        if value == "root":
-            return queryset.filter(parent_snapshot__isnull=True)
-        if value == "discovered":
-            return queryset.filter(parent_snapshot__isnull=False)
-        if value in {"has_children", "no_children"}:
-            child_snapshots = Snapshot.objects.filter(parent_snapshot_id=OuterRef("pk"))
-            queryset = queryset.annotate(has_child_snapshots=Exists(child_snapshots))
-            return queryset.filter(has_child_snapshots=value == "has_children")
-        return queryset
-
-
 class SnapshotArchiveStateListFilter(admin.SimpleListFilter):
     title = "archive state"
     parameter_name = "archive_state"
@@ -242,31 +187,10 @@ class SnapshotSizeListFilter(admin.SimpleListFilter):
         return queryset
 
 
-class SnapshotRetryListFilter(admin.SimpleListFilter):
-    title = "retry"
-    parameter_name = "retry"
-
-    def lookups(self, request, model_admin):
-        return (
-            ("due", "Due now"),
-            ("future", "Scheduled later"),
-            ("none", "No retry time"),
-        )
-
-    def queryset(self, request, queryset):
-        value = self.value()
-        if value == "due":
-            return queryset.filter(retry_at__isnull=False, retry_at__lte=timezone.now())
-        if value == "future":
-            return queryset.filter(retry_at__gt=timezone.now())
-        if value == "none":
-            return queryset.filter(retry_at__isnull=True)
-        return queryset
-
-
 class SnapshotResultHealthListFilter(admin.SimpleListFilter):
     title = "ArchiveResult status"
     parameter_name = "archiveresult_status"
+    SNAPSHOT_FIRST_VALUES = {"succeeded"}
 
     def lookups(self, request, model_admin):
         return (
@@ -276,7 +200,6 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
             ("failed", ">50% failed"),
             ("running", ">50% running"),
             ("pending", ">50% queued"),
-            ("backoff", ">50% waiting to retry"),
             ("noresults", ">50% noresults"),
         )
 
@@ -292,60 +215,68 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
                 "succeeded": ArchiveResult.StatusChoices.SUCCEEDED,
                 "failed": ArchiveResult.StatusChoices.FAILED,
                 "running": ArchiveResult.StatusChoices.STARTED,
-                "pending": ArchiveResult.StatusChoices.QUEUED,
+                "pending": (ArchiveResult.StatusChoices.QUEUED, ArchiveResult.StatusChoices.BACKOFF),
                 "backoff": ArchiveResult.StatusChoices.BACKOFF,
                 "noresults": ArchiveResult.StatusChoices.NORESULTS,
             }
             if value in status_by_value:
-                queryset = queryset.annotate(
-                    total_results=Count("archiveresult"),
-                    matching_results=Count(
-                        "archiveresult",
-                        filter=Q(archiveresult__status=status_by_value[value]),
-                    ),
-                )
-                return queryset.filter(matching_results__gt=F("total_results") / 2)
+                status = status_by_value[value]
+                if value in self.SNAPSHOT_FIRST_VALUES:
+                    # "succeeded" is overwhelmingly common in large collections.
+                    # Scan Snapshots in admin order and do indexed per-snapshot
+                    # probes so page 1 can stop after list_per_page matches.
+                    queryset = queryset.alias(
+                        total_results=ArchiveResult.snapshot_count_expr(),
+                        matching_results=ArchiveResult.snapshot_count_expr(status=status),
+                    ).filter(matching_results__gt=F("total_results") / 2)
+                    return queryset
+
+                # Rare statuses are faster status-first: use the
+                # (status, snapshot_id) index to find candidate snapshots.
+                snapshot_ids = ArchiveResult.cached_snapshot_ids_with_majority_status(status)
+                queryset = queryset.filter(pk__in=snapshot_ids)
+                queryset._archivebox_count_hint = len(snapshot_ids)
+                queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
+                return queryset
         return queryset
 
 
 class SnapshotChangeList(SearchResultsChangeList):
     def __init__(self, request, *args, **kwargs):
         super().__init__(request, *args, **kwargs)
-        resolver_name = getattr(getattr(request, "resolver_match", None), "url_name", "")
+        resolver_name = request.resolver_match.url_name
         self.embedded_changelist = request.GET.get("_embedded") == "crawl"
         self.snapshot_is_grid_view = not self.embedded_changelist and (
             resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")
         )
 
+    def _attach_archiveresult_summaries(self):
+        snapshot_ids = [obj.pk for obj in self.result_list]
+        if not snapshot_ids:
+            return
+
+        results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
+        seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids}
+        rows = (
+            ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
+            .order_by("snapshot_id", "plugin")
+            .values_list("snapshot_id", "plugin", "status", "output_size")
+        )
+        for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000):
+            if plugin in seen_plugins[snapshot_id]:
+                continue
+            seen_plugins[snapshot_id].add(plugin)
+            results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size))
+
+        for obj in self.result_list:
+            obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
+
     def get_results(self, request):
         super().get_results(request)
         if request.GET.get("_embedded") == "crawl":
             self.full_result_count = self.result_count
-        else:
-            self.full_result_count = self.model_admin.get_paginator(
-                request,
-                self.model._default_manager.all().order_by(),
-                self.list_per_page,
-            ).count
-        self.show_full_result_count = True
-
-        snapshot_ids = [obj.pk for obj in self.result_list]
-        if snapshot_ids:
-            results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
-            seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids}
-            rows = (
-                ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
-                .order_by("snapshot_id", "plugin")
-                .values_list("snapshot_id", "plugin", "status", "output_size")
-            )
-            for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000):
-                if plugin in seen_plugins[snapshot_id]:
-                    continue
-                seen_plugins[snapshot_id].add(plugin)
-                results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size))
-
-            for obj in self.result_list:
-                obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
+            self.show_full_result_count = True
+        self._attach_archiveresult_summaries()
 
 
 class SnapshotAdminForm(forms.ModelForm):
@@ -381,18 +312,13 @@ class SnapshotAdminForm(forms.ModelForm):
         instance = super().save(commit=False)
         permissions = self.cleaned_data["permissions_config"]
         config = dict(instance.config or {})
-        if permissions == instance.crawl.permissions:
-            config.pop("PERMISSIONS", None)
-        else:
-            config["PERMISSIONS"] = permissions
+        config["PERMISSIONS"] = permissions
         instance.config = config
 
         # Handle tags_editor field
         if commit:
             instance.save()
-            save_m2m = getattr(self, "_save_m2m", None)
-            if callable(save_m2m):
-                save_m2m()
+            self._save_m2m()
 
             # Parse and save tags from tags_editor
             tags_str = self.cleaned_data.get("tags_editor", "")
@@ -446,11 +372,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         SnapshotPermissionsListFilter,
         SnapshotStatusListFilter,
         SnapshotResultHealthListFilter,
-        SnapshotDepthListFilter,
-        SnapshotRelationListFilter,
         SnapshotArchiveStateListFilter,
         SnapshotSizeListFilter,
-        SnapshotRetryListFilter,
         "created_at",
         "downloaded_at",
         "crawl__created_by",
@@ -531,8 +454,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         ),
     )
 
-    ordering = ["-timestamp"]
-    actions = ["add_tags", "remove_tags", "resnapshot_snapshot", "update_snapshots", "overwrite_snapshots", "delete_snapshots"]
+    ordering = ["-created_at"]
+    actions = [
+        "add_tags",
+        "remove_tags",
+        "resnapshot_snapshot",
+        "update_snapshots",
+        "overwrite_snapshots",
+        "set_snapshot_permissions",
+        "delete_snapshots",
+    ]
     inlines = []  # Removed TagInline, using TagEditorWidget instead
     list_per_page = 50
 
@@ -540,7 +471,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     paginator = AcceleratedPaginator
 
     save_on_top = True
-    show_full_result_count = False
+    show_full_result_count = True
 
     def get_changelist(self, request, **kwargs):
         return SnapshotChangeList
@@ -551,14 +482,21 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return super().get_ordering(request)
 
     def change_view(self, request, object_id, form_url="", extra_context=None):
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
+        self.request = request
         extra_context = extra_context or {}
         extra_context["CONFIG"] = request.archivebox_config
+        snapshot = self.get_object(request, object_id)
+        if snapshot and snapshot.status in {
+            Snapshot.StatusChoices.QUEUED,
+            Snapshot.StatusChoices.STARTED,
+            Snapshot.StatusChoices.PAUSED,
+        }:
+            extra_context["progress_auto_expand"] = True
+            extra_context["progress_endpoint"] = progress_endpoint("snapshot", snapshot.id)
         return super().change_view(request, object_id, form_url, extra_context | GLOBAL_CONTEXT)
 
     def changelist_view(self, request, extra_context=None):
         self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         saved_list_per_page = self.list_per_page
         embedded_changelist = request.GET.get("_embedded") == "crawl"
         if embedded_changelist:
@@ -594,12 +532,12 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return super().lookup_allowed(lookup, value, request=request)
 
     def get_snapshot_view_url(self, obj: Snapshot) -> str:
-        request = getattr(self, "request", None)
-        return build_snapshot_url(str(obj.id), request=request, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)
 
     def get_snapshot_files_url(self, obj: Snapshot) -> str:
-        request = getattr(self, "request", None)
-        return f"{build_snapshot_url(str(obj.id), request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
+        request = self.request
+        return f"{build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)}/?files=1"
 
     def get_snapshot_zip_url(self, obj: Snapshot) -> str:
         return f"{self.get_snapshot_files_url(obj)}&download=zip"
@@ -619,155 +557,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return custom_urls + urls
 
     def search_stream_view(self, request):
-        from archivebox.search import iter_query_search_ids
-
-        query = (request.GET.get("q") or "").strip()
-        from archivebox.search import get_search_mode, get_search_mode_base
-
-        search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
-        if not query:
-            return StreamingHttpResponse((), content_type="text/plain")
-
-        search_url = request.GET.get("search_url") or request.get_full_path()
-        target_url = urlsplit(search_url)
-        target_get = QueryDict(target_url.query, mutable=True)
-        for key in ("q", "search_mode", "p", "search_url"):
-            target_get.pop(key, None)
-
-        filter_request = copy(request)
-        filter_request.path = target_url.path or request.path
-        filter_request.path_info = target_url.path or request.path_info
-        filter_request.GET = target_get
-        filter_request.archivebox_config = getattr(request, "archivebox_config", None)
-
-        # Build the same filtered base queryset the changelist uses, but with
-        # the search params stripped. The stream then intersects each wave with
-        # this queryset before writing IDs into the short-lived cache.
-        current_request = getattr(self, "request", None)
-        try:
-            base_queryset = self.get_changelist_instance(filter_request).queryset
-        finally:
-            self.request = current_request
-
-        async def snapshot_ids():
-            seen = set()
-            ids = []
-            last_sent = 0
-            stream_batch_size = 100
-            stream_padding = " " * 4096
-            cache_key = get_admin_search_cache_key(request, search_url)
-            cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-            yield f"0{stream_padding}\n"
-            queue = Queue(maxsize=8)
-            stop_event = threading.Event()
-
-            def emit(item):
-                while not stop_event.is_set():
-                    try:
-                        queue.put(item, timeout=0.1)
-                        return
-                    except Full:
-                        continue
-
-            def run_search():
-                nonlocal last_sent
-                iterator = None
-                try:
-                    search_mode_base = get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None))
-                    iterator = (
-                        self.iter_meta_search_ids(query, base_queryset)
-                        if search_mode_base == "meta"
-                        else self.iter_backend_search_ids(
-                            iter_query_search_ids(query, search_mode=search_mode, config=getattr(request, "archivebox_config", None)),
-                            base_queryset,
-                        )
-                    )
-                    for snapshot_id in iterator:
-                        if stop_event.is_set():
-                            break
-                        snapshot_id = str(snapshot_id).strip().lower()
-                        if len(snapshot_id.replace("-", "")) != 32 or snapshot_id in seen:
-                            continue
-                        seen.add(snapshot_id)
-                        ids.append(snapshot_id)
-                        if len(ids) - last_sent >= stream_batch_size:
-                            cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-                            last_sent = len(ids)
-                            emit(f"{last_sent}{stream_padding}\n")
-                    if not stop_event.is_set() and len(ids) != last_sent:
-                        cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-                        emit(f"{len(ids)}{stream_padding}\n")
-                except BaseException as err:
-                    emit(err)
-                finally:
-                    if iterator is not None:
-                        try:
-                            iterator.close()
-                        except AttributeError:
-                            pass
-                    cache.set(cache_key, {"ids": ids, "done": True}, SEARCH_RESULT_CACHE_TTL)
-                    emit(None)
-
-            threading.Thread(target=run_search, name="admin-snapshot-search-stream", daemon=True).start()
-            try:
-                while True:
-                    item = await asyncio.to_thread(queue.get)
-                    if item is None:
-                        break
-                    if isinstance(item, BaseException):
-                        raise item
-                    yield item
-            finally:
-                stop_event.set()
-
-        response = StreamingHttpResponse(snapshot_ids(), content_type="text/plain")
-        response["X-Accel-Buffering"] = "no"
-        return response
-
-    def iter_meta_search_ids(self, query, queryset):
-        seen = set()
-        try:
-            snapshot_id = UUID(query)
-        except ValueError:
-            snapshot_id = None
-        if snapshot_id:
-            for pk in queryset.filter(pk=snapshot_id).values_list("pk", flat=True):
-                seen.add(pk)
-                yield pk
-
-        for wave in (
-            Q(timestamp__startswith=query) | Q(url__istartswith=query) | Q(title__istartswith=query),
-            Q(url__icontains=query),
-            Q(title__icontains=query),
-            Q(tags__name__icontains=query),
-        ):
-            for pk in queryset.filter(wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500):
-                if pk in seen:
-                    continue
-                seen.add(pk)
-                yield pk
-
-    def iter_backend_search_ids(self, iterator, queryset):
-        batch = []
-        seen = set()
-
-        def flush_batch():
-            valid = {str(pk) for pk in queryset.filter(pk__in=batch).values_list("pk", flat=True)}
-            for snapshot_id in batch:
-                if snapshot_id in valid and snapshot_id not in seen:
-                    seen.add(snapshot_id)
-                    yield snapshot_id
-
-        for snapshot_id in iterator:
-            snapshot_id = str(snapshot_id).strip().lower()
-            if len(snapshot_id.replace("-", "")) != 32:
-                continue
-            batch.append(snapshot_id)
-            if len(batch) >= 200:
-                yield from flush_batch()
-                batch = []
-        if batch:
-            yield from flush_batch()
+        return admin_snapshot_search_stream_view(self, request)
 
     def set_permissions_view(self, request, object_id):
         if request.method != "POST":
@@ -777,12 +567,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         if permissions not in dict(PERMISSIONS_CHOICES):
             return HttpResponseBadRequest("Invalid permissions value")
 
-        snapshot = get_object_or_404(Snapshot.objects.select_related("crawl"), pk=object_id)
+        snapshot = get_object_or_404(Snapshot, pk=object_id)
         config = dict(snapshot.config or {})
-        if permissions == snapshot.crawl.permissions:
-            config.pop("PERMISSIONS", None)
-        else:
-            config["PERMISSIONS"] = permissions
+        config["PERMISSIONS"] = permissions
 
         # Keep the quick-edit write to one targeted UPDATE so SQLite only holds
         # the write lock for the permission/config change itself. safe_update()
@@ -792,6 +579,35 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
         return JsonResponse({"permissions": permissions, "icon": icon, "label": label, "fg": fg, "bg": bg})
 
+    @admin.action(description="Permissions ▾")
+    def set_snapshot_permissions(self, request, queryset):
+        permissions = (request.POST.get("permissions") or "").strip().lower()
+        if permissions not in dict(PERMISSIONS_CHOICES):
+            messages.error(request, "Choose a valid permissions value.")
+            return
+        updated = self.update_snapshot_permissions(queryset, permissions)
+        messages.success(request, f"Set permissions to {permissions} on {updated} snapshot(s).")
+
+    def update_snapshot_permissions(self, queryset, permissions):
+        now = timezone.now()
+        updated = 0
+        batch = []
+        snapshots = queryset.select_related(None).only("id", "config").prefetch_related(None)
+        for snapshot in snapshots.iterator(chunk_size=500):
+            config = dict(snapshot.config or {})
+            config["PERMISSIONS"] = permissions
+            snapshot.config = config
+            snapshot.modified_at = now
+            batch.append(snapshot)
+            if len(batch) >= 500:
+                Snapshot.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
+                updated += len(batch)
+                batch.clear()
+        if batch:
+            Snapshot.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
+            updated += len(batch)
+        return updated
+
     def redo_failed_view(self, request, object_id):
         snapshot = get_object_or_404(Snapshot, pk=object_id)
 
@@ -810,31 +626,13 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
         return redirect(snapshot.admin_change_url)
 
-    # def get_queryset(self, request):
-    #     # tags_qs = SnapshotTag.objects.all().select_related('tag')
-    #     # prefetch = Prefetch('snapshottag_set', queryset=tags_qs)
-
-    #     self.request = request
-    #     return super().get_queryset(request).prefetch_related('archiveresult_set').distinct()  # .annotate(archiveresult_count=Count('archiveresult'))
     def get_queryset(self, request):
         self.request = request
         ordering_fields = self._get_ordering_fields(request)
         needs_files_sort = "files" in ordering_fields
         needs_tags_sort = "tags_inline" in ordering_fields
-        is_change_view = getattr(getattr(request, "resolver_match", None), "url_name", "") == "core_snapshot_change"
-        prefetches = [
-            Prefetch(
-                "crawl",
-                queryset=Crawl.objects.only(
-                    "id",
-                    "permissions",
-                    "persona_id",
-                    "status",
-                    "created_by_id",
-                ).prefetch_related("created_by"),
-            ),
-            "tags",
-        ]
+        is_change_view = request.resolver_match.url_name == "core_snapshot_change"
+        prefetches = ["tags"]
         if is_change_view:
             prefetches.append(
                 Prefetch(
@@ -845,13 +643,26 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                         "plugin",
                         "status",
                         "output_size",
+                        "output_files",
+                    ),
+                ),
+            )
+        else:
+            prefetches.append(
+                Prefetch(
+                    "crawl",
+                    queryset=Crawl.objects.select_related("created_by").only(
+                        "id",
+                        "created_by_id",
+                        "created_by__id",
+                        "created_by__username",
                     ),
                 ),
             )
 
         qs = super().get_queryset(request)
         if is_change_view:
-            qs = qs.defer("notes")
+            qs = qs.select_related("crawl__created_by").defer("notes")
         else:
             qs = qs.only(
                 "id",
@@ -869,10 +680,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         qs = qs.prefetch_related(*prefetches)
         if needs_files_sort:
             qs = qs.annotate(
-                ar_succeeded_count=Count(
-                    "archiveresult",
-                    filter=Q(archiveresult__status="succeeded"),
-                ),
+                ar_succeeded_count=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
             )
         if needs_tags_sort:
             qs = qs.annotate(tag_count=Count("tags", distinct=True))
@@ -881,12 +689,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="👁", ordering="permissions")
     def permissions_badge(self, obj):
-        permissions = getattr(obj, "snapshot_permissions", None)
+        permissions = obj.__dict__.get("snapshot_permissions")
         if permissions is None:
-            if obj.permissions:
-                permissions = obj.permissions
-            else:
-                permissions = obj.crawl.permissions
+            permissions = obj.permissions
         permissions = normalize_permissions(permissions)
         icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
         menu_items = format_html_join(
@@ -1051,8 +856,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
 
     def status_info(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         favicon_url = build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)
         return format_html(
             """
@@ -1069,16 +874,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Archive Results")
     def archiveresults_list(self, obj):
-        request = getattr(self, "request", None)
-        return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=request.archivebox_config)
 
     @admin.display(
         description="Title",
         ordering="title",
     )
     def title_str(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         title_raw = (obj.title or "").strip()
         url_raw = (obj.url or "").strip()
         title_normalized = title_raw.lower()
@@ -1131,34 +936,58 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return mark_safe(f'<span class="tags-inline-editor">{tags_html}</span>')
 
     def _get_preview_data(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         results = self._get_prefetched_results(obj)
         if results is not None:
-            has_screenshot = any(r.plugin == "screenshot" for r in results)
-            has_favicon = any(r.plugin == "favicon" for r in results)
+            results = [r for r in results if r.plugin in ("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")]
         else:
-            available_plugins = set(obj.archiveresult_set.filter(plugin__in=("screenshot", "favicon")).values_list("plugin", flat=True))
-            has_screenshot = "screenshot" in available_plugins
-            has_favicon = "favicon" in available_plugins
+            results = list(
+                obj.archiveresult_set.filter(plugin__in=("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")).only(
+                    "snapshot_id",
+                    "plugin",
+                    "status",
+                    "output_files",
+                    "output_str",
+                ),
+            )
 
-        if not has_screenshot and not has_favicon:
+        def result_output_path(result, filename: str) -> str | None:
+            output_files = result.output_files or {}
+            file_info = output_files.get(filename)
+            if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
+                return None
+            if file_info.get("root_relative"):
+                return filename
+            return f"{result.plugin}/{filename}"
+
+        def result_urls(plugin: str, filenames: tuple[str, ...]) -> list[str]:
+            urls: list[str] = []
+            for result in results:
+                if result.plugin != plugin or result.status != ArchiveResult.StatusChoices.SUCCEEDED:
+                    continue
+                for filename in filenames:
+                    output_path = result_output_path(result, filename)
+                    if output_path:
+                        urls.append(build_snapshot_url(str(obj.id), output_path, request=request, config=config))
+            return urls
+
+        screenshot_urls = result_urls("screenshot", ("screenshot.png",))
+        extension_screenshot_urls = result_urls(EXTENSION_SCREENSHOT_PLUGIN, ("screenshot-1.png", "screenshot.png"))
+        favicon_urls = result_urls("favicon", ("favicon.ico",))
+
+        if not screenshot_urls and not extension_screenshot_urls and not favicon_urls:
             return None
 
-        if has_screenshot:
-            img_url = build_snapshot_url(str(obj.id), "screenshot/screenshot.png", request=request, config=config)
-            fallbacks = [
-                build_snapshot_url(str(obj.id), "screenshot.png", request=request, config=config),
-                build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config),
-                build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config),
-            ]
+        all_screenshot_urls = [*screenshot_urls, *extension_screenshot_urls]
+        if all_screenshot_urls:
+            img_url = all_screenshot_urls[0]
+            fallbacks = [*all_screenshot_urls[1:], *favicon_urls]
             img_alt = "Screenshot"
             preview_class = "screenshot"
         else:
-            img_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
-            fallbacks = [
-                build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config),
-            ]
+            img_url = favicon_urls[0]
+            fallbacks = favicon_urls[1:]
             img_alt = "Favicon"
             preview_class = "favicon"
 
@@ -1176,6 +1005,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             "preview_class": preview_class,
             "onerror_js": onerror_js,
             "fallback_list": fallback_list,
+            "favicon_url": favicon_urls[0] if favicon_urls else "",
+            "favicon_fallback_list": ",".join(favicon_urls[1:]),
         }
 
     @admin.display(description="", empty_value="")
@@ -1184,10 +1015,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         if not preview:
             return ""
 
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
-        favicon_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
-        fallback_list = ",".join([build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)])
+        favicon_url = preview.get("favicon_url") or ""
+        if not favicon_url:
+            return ""
+        fallback_list = preview.get("favicon_fallback_list") or ""
         onerror_js = (
             "this.dataset.fallbacks && this.dataset.fallbacks.length ? "
             "(this.src=this.dataset.fallbacks.split(',').shift(), "
@@ -1223,8 +1054,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description=" ", empty_value="")
     def snapshot_summary(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         preview = self._get_preview_data(obj)
         stats = self._get_progress_stats(obj)
         archive_size = stats["output_size"] or 0
@@ -1288,8 +1119,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
         visible_results = sorted_results[:14]
         output = []
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         for result in visible_results:
             icon = mark_safe(get_plugin_icon(result.plugin))
             if not icon.strip():
@@ -1320,8 +1151,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         # ordering='archiveresult_count'
     )
     def size(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         archive_size = self._get_progress_stats(obj)["output_size"] or 0
         if archive_size:
             size_txt = printable_filesize(archive_size)
@@ -1435,7 +1266,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
 
     def _get_progress_stats(self, obj):
-        cached_stats = getattr(obj, "_admin_progress_stats", None)
+        cached_stats = obj.__dict__.get("_admin_progress_stats")
         if cached_stats is not None:
             return cached_stats
 
@@ -1481,39 +1312,38 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def _get_prefetched_results(self, obj):
         if "_admin_archiveresults" in obj.__dict__:
             return obj.__dict__["_admin_archiveresults"]
-        if hasattr(obj, "_prefetched_objects_cache") and "archiveresult_set" in obj._prefetched_objects_cache:
+        if "archiveresult_set" in obj.__dict__.get("_prefetched_objects_cache", {}):
             return obj.archiveresult_set.all()
         return None
 
     def _get_expected_hook_total(self, obj) -> int:
         try:
-            request = getattr(self, "request", None)
-            if getattr(getattr(request, "resolver_match", None), "url_name", "") in {"core_snapshot_changelist", "core_snapshot_change"}:
+            request = self.request
+            if request.resolver_match.url_name in {"core_snapshot_changelist", "core_snapshot_change"}:
                 return 0
 
-            crawl = getattr(obj, "crawl", None)
-            snapshot_config = getattr(obj, "config", None) or {}
-            crawl_config = getattr(crawl, "config", None) or {}
-            crawl_persona_id = getattr(crawl, "persona_id", None)
-            has_scoped_config = bool(snapshot_config or crawl_config or crawl_persona_id)
+            crawl = obj.crawl
+            snapshot_config = obj.config or {}
+            crawl_config = crawl.config or {}
+            has_scoped_config = bool(snapshot_config or crawl_config)
 
             if request is not None and not has_scoped_config:
-                cached_total = getattr(request, "archivebox_expected_snapshot_hook_total", None)
+                cached_total = request.__dict__.get("archivebox_expected_snapshot_hook_total")
                 if cached_total is None:
-                    config = getattr(request, "archivebox_config", None) or get_config()
+                    config = request.archivebox_config
                     cached_total = len(discover_hooks("Snapshot", config=config))
                     request.archivebox_expected_snapshot_hook_total = cached_total
                 return cached_total
 
             if request is not None:
-                scoped_cache = getattr(request, "archivebox_expected_snapshot_hook_totals_by_scope", None)
+                scoped_cache = request.__dict__.get("archivebox_expected_snapshot_hook_totals_by_scope")
                 if scoped_cache is None:
                     scoped_cache = {}
                     request.archivebox_expected_snapshot_hook_totals_by_scope = scoped_cache
                 if snapshot_config:
                     cache_key = ("snapshot", json.dumps(snapshot_config, sort_keys=True, default=str))
                 else:
-                    cache_key = ("crawl", json.dumps(crawl_config, sort_keys=True, default=str), crawl_persona_id)
+                    cache_key = ("crawl", json.dumps(crawl_config, sort_keys=True, default=str))
                 cached_total = scoped_cache.get(cache_key)
                 if cached_total is None:
                     config = get_config(crawl=crawl, snapshot=obj if snapshot_config else None)
@@ -1526,8 +1356,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             return 0
 
     def _get_prefetched_tags(self, obj):
-        if hasattr(obj, "_prefetched_objects_cache") and "tags" in obj._prefetched_objects_cache:
-            return list(obj._prefetched_objects_cache["tags"])
+        prefetched_cache = obj.__dict__.get("_prefetched_objects_cache", {})
+        if "tags" in prefetched_cache:
+            return list(prefetched_cache["tags"])
         return None
 
     def _get_ordering_fields(self, request):
@@ -1620,7 +1451,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         # work-in-progress crawl, not the old snapshot they re-archived from.
         # A snapshot-view redirect would race the runner — the new snapshot
         # may sit queued for a while before the runner creates the DB row.
-        return redirect(f"/admin/crawls/crawl/{crawl.id.hex}/change/#snapshots")
+        return redirect(f"/admin/crawls/crawl/{crawl.id}/change/#snapshots")
 
     @admin.action(
         description="🔄 Redo",

archivebox/core/admin_tags.py

@@ -24,7 +24,7 @@ from archivebox.core.tag_util import (
     normalize_has_snapshots_filter,
     normalize_tag_sort,
 )
-from archivebox.core.host_util import build_snapshot_url
+from archivebox.core.routes_util import build_snapshot_url
 
 
 class TagInline(admin.TabularInline):
@@ -117,7 +117,7 @@ class TagAdmin(BaseModelAdmin):
         sort = normalize_tag_sort((request.GET.get("sort") or "created_desc").strip())
         created_by = normalize_created_by_filter((request.GET.get("created_by") or "").strip())
         year = normalize_created_year_filter((request.GET.get("year") or "").strip())
-        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "yes").strip())
+        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "all").strip())
         context = {
             **self.admin_site.each_context(request),
             **(extra_context or {}),
@@ -231,7 +231,7 @@ class TagAdmin(BaseModelAdmin):
 
     @admin.display(description="Snapshots", ordering="num_snapshots")
     def num_snapshots(self, tag: Tag):
-        count = getattr(tag, "num_snapshots", None)
+        count = tag.__dict__.get("num_snapshots")
         if count is None:
             count = tag.snapshot_set.count()
         return format_html(

archivebox/core/admin_users.py

@@ -47,7 +47,7 @@ class CustomUserAdmin(UserAdmin):
 
     def snapshot_count_badge(self, obj):
         snapshots_url = f"/admin/core/snapshot/?created_by__id__exact={obj.pk}"
-        snapshot_count = getattr(obj, "snapshot_count", 0)
+        snapshot_count = obj.__dict__.get("snapshot_count", 0)
         snapshot_label = "snapshot" if snapshot_count == 1 else "snapshots"
         return format_html(
             (

archivebox/core/forms.py

@@ -1,23 +1,23 @@
 __package__ = "archivebox.core"
 
-import json
 import re
-from collections.abc import Iterable, Mapping
 from decimal import Decimal, InvalidOperation, ROUND_CEILING
-from pathlib import Path
-from typing import Any
 
 from django import forms
-from django.utils.html import format_html
 
 from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
-from taggit.utils import edit_string_for_tags, parse_tags
 from archivebox.base_models.admin import KeyValueWidget
 from archivebox.crawls.schedule_util import validate_schedule
 from archivebox.config.common import get_config, parse_delete_after
-from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions, is_admin_user
+from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
-from archivebox.hooks import get_plugins, discover_plugin_configs, get_plugin_icon
+from archivebox.plugins.discovery import get_plugins
+from archivebox.plugins.forms import (
+    PLUGIN_GROUP_DEFINITIONS,
+    TIMEOUT_INPUT_PATTERN,
+    PluginConfigFormMixin,
+    get_choice_field,
+)
 from archivebox.personas.models import Persona
 
 DEPTH_CHOICES = (
@@ -28,605 +28,85 @@ DEPTH_CHOICES = (
     ("4", "depth = 4 (+ URLs four hops away)"),
 )
 
-PLUGIN_CONFIG_FIELD_PREFIX = "plugin_config__"
-PLUGIN_GROUP_DEFINITIONS = (
-    (
-        "main_plugins",
-        "Main",
-        "",
-        "",
-        "",
-        (
-            "dom",
-            "screenshot",
-            "pdf",
-            "singlefile",
-            "wget",
-            "archivedotorg",
-            "chrome_mhtml",
-            "archivewebpage",
-        ),
-    ),
-    (
-        "page_setup_plugins",
-        "Page Setup",
-        "",
-        "",
-        "",
-        (
-            "chrome",
-            "infiniscroll",
-            "modalcloser",
-            "ublock",
-            "istilldontcareaboutcookies",
-            "twocaptcha",
-            "claudechrome",
-        ),
-    ),
-    (
-        "media_plugins",
-        "Media",
-        "",
-        "",
-        "",
-        (
-            "staticfile",
-            "responses",
-            "chrome_screencast",
-            "ytdlp",
-            "gallerydl",
-            "git",
-        ),
-    ),
-    (
-        "text_plugins",
-        "Text",
-        "",
-        "",
-        "",
-        (
-            "readability",
-            "htmltotext",
-            "defuddle",
-            "forumdl",
-            "mercury",
-            "trafilatura",
-            "liteparse",
-            "opendataloader",
-            "papersdl",
-        ),
-    ),
-    (
-        "metadata_plugins",
-        "Metadata",
-        "",
-        "",
-        "",
-        (
-            "title",
-            "favicon",
-            "headers",
-            "redirects",
-            "accessibility",
-            "consolelog",
-            "sslcerts",
-            "dns",
-            "seo",
-            "hashes",
-        ),
-    ),
-    (
-        "postprocessing_plugins",
-        "Postprocessing",
-        "",
-        "",
-        "",
-        (
-            "parse_dom_outlinks",
-            "parse_html_urls",
-            "parse_jsonl_urls",
-            "parse_netscape_urls",
-            "parse_rss_urls",
-            "parse_txt_urls",
-            "claudecode",
-            "claudecodecleanup",
-            "claudecodeextract",
-        ),
-    ),
-)
-HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
-    "apt",
-    "base",
-    "bash",
-    "brew",
-    "cargo",
-    "chromewebstore",
-    "env",
-    "media",
-    "npm",
-    "pip",
-    "puppeteer",
-    "search_backend_ripgrep",
-    "search_backend_sonic",
-    "search_backend_sqlite",
-    "ssl",
-}
-TIMEOUT_INPUT_PATTERN = r"(0|[1-9][0-9]*|[0-9]+(?:\.[0-9]+)?\s*(?:s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours))"
 
-
-def get_plugin_choices():
-    """Get available extractor plugins from discovered hooks."""
-    return [(name, name) for name in get_plugins()]
-
-
-def get_plugin_choice_label(plugin_name: str, plugin_configs: dict[str, dict]) -> str:
-    schema = plugin_configs.get(plugin_name, {})
-    description = str(schema.get("description") or "").strip()
-    if not description:
-        return plugin_name
-    icon_html = get_plugin_icon(plugin_name)
-
-    return format_html(
-        '<span class="plugin-choice-icon">{}</span><span class="plugin-choice-name">{}</span>',
-        icon_html,
-        plugin_name,
-    )
-
-
-def get_choice_field(form: forms.Form, name: str) -> forms.ChoiceField:
-    field = form.fields[name]
-    if not isinstance(field, forms.ChoiceField):
-        raise TypeError(f"{name} must be a ChoiceField")
-    return field
-
-
-def _plugin_config_input_name(plugin_name: str, config_key: str) -> str:
-    return f"{PLUGIN_CONFIG_FIELD_PREFIX}{plugin_name}__{config_key}"
-
-
-def _schema_types(schema: Mapping[str, Any]) -> list[str]:
-    raw_type = schema.get("type") or "string"
-    if isinstance(raw_type, list):
-        return [str(item) for item in raw_type]
-    return [str(raw_type)]
-
-
-def _jsonish(value: Any) -> str:
-    if isinstance(value, str):
-        return value
-    return json.dumps(value, sort_keys=True, default=str)
-
-
-def _same_config_value(left: Any, right: Any) -> bool:
-    return json.dumps(left, sort_keys=True, default=str) == json.dumps(right, sort_keys=True, default=str)
-
-
-def _coerce_plugin_config_value(raw_value: Any, schema: Mapping[str, Any]) -> Any:
-    schema_types = _schema_types(schema)
-
-    if "boolean" in schema_types:
-        if isinstance(raw_value, bool):
-            return raw_value
-        value = str(raw_value).strip().lower()
-        if value in {"true", "1", "yes", "on"}:
-            return True
-        if value in {"false", "0", "no", "off", ""}:
-            return False
-        raise forms.ValidationError("Must be true or false.")
-
-    if "integer" in schema_types:
-        value = int(str(raw_value).strip())
-        minimum = schema.get("minimum")
-        maximum = schema.get("maximum")
-        if minimum is not None and value < int(minimum):
-            raise forms.ValidationError(f"Must be at least {minimum}.")
-        if maximum is not None and value > int(maximum):
-            raise forms.ValidationError(f"Must be at most {maximum}.")
-        return value
-
-    if "number" in schema_types:
-        value = float(str(raw_value).strip())
-        minimum = schema.get("minimum")
-        maximum = schema.get("maximum")
-        if minimum is not None and value < float(minimum):
-            raise forms.ValidationError(f"Must be at least {minimum}.")
-        if maximum is not None and value > float(maximum):
-            raise forms.ValidationError(f"Must be at most {maximum}.")
-        return value
-
-    if "array" in schema_types:
-        if isinstance(raw_value, list):
-            return raw_value
-        value = str(raw_value).strip()
-        if not value:
-            return []
-        if value.startswith("["):
-            parsed = json.loads(value)
-            if not isinstance(parsed, list):
-                raise forms.ValidationError("Must be a JSON array.")
-            return parsed
-        return [item.strip() for item in value.replace(",", "\n").splitlines() if item.strip()]
-
-    if "object" in schema_types:
-        value = str(raw_value).strip()
-        if not value:
-            return {}
-        parsed = json.loads(value)
-        if not isinstance(parsed, dict):
-            raise forms.ValidationError("Must be a JSON object.")
-        return parsed
-
-    value = str(raw_value)
-    enum = schema.get("enum")
-    if isinstance(enum, list) and enum and value not in {str(item) for item in enum}:
-        raise forms.ValidationError(f"Must be one of: {', '.join(str(item) for item in enum)}.")
-    return value
-
-
-class PluginConfigFormMixin:
-    plugin_groups: list[dict[str, Any]]
-
-    def build_plugin_groups(self, runtime_config: Mapping[str, Any] | None = None) -> None:
-        all_plugins = get_plugins()
-        plugin_configs = discover_plugin_configs()
-        runtime_config = runtime_config or get_config()
-        self.plugin_config_binary_urls = get_plugin_config_binary_urls(runtime_config)
-        grouped_plugins = set().union(*(group[-1] for group in PLUGIN_GROUP_DEFINITIONS))
-        other_plugins = tuple(sorted(set(all_plugins) - grouped_plugins - HIDDEN_PLUGIN_CONFIG_UI_PLUGINS))
-
-        for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
-            if field_name in self.fields:
-                get_choice_field(self, field_name).choices = [
-                    (p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins
-                ]
-
-        if "other_plugins" in self.fields:
-            get_choice_field(self, "other_plugins").choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in other_plugins]
-
-        group_specs = (
-            *PLUGIN_GROUP_DEFINITIONS,
-            ("other_plugins", "Other", "", "", "", other_plugins),
-        )
-        binary_url_lookup = _build_required_binary_url_lookup(plugin_configs, runtime_config)
-        self.plugin_groups = [
-            {
-                "field_name": field_name,
-                "title": title,
-                "note": note,
-                "dom_id": dom_id,
-                "select_all_group": select_all_group,
-                "show_selectors": field_name in self.fields,
-                "plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config, binary_url_lookup),
-            }
-            for field_name, title, note, dom_id, select_all_group, plugin_names in group_specs
-            if any(plugin in all_plugins for plugin in plugin_names)
-        ]
-
-    def _build_plugin_cards(
-        self,
-        field_name: str,
-        plugin_names: Iterable[str],
-        plugin_configs: dict[str, dict[str, Any]],
-        runtime_config: Mapping[str, Any],
-        binary_url_lookup: Mapping[str, str] | None = None,
-    ) -> list[dict[str, Any]]:
-        if field_name in self.fields:
-            choices = list(get_choice_field(self, field_name).choices)
-            selected_values = set(self.data.getlist(field_name)) if self.is_bound else set(get_choice_field(self, field_name).initial or [])
-        else:
-            all_plugins = get_plugins()
-            choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins]
-            selected_values = set()
-
-        cards = []
-        for index, (plugin_name, label) in enumerate(choices):
-            schema = plugin_configs.get(str(plugin_name), {})
-            properties = schema.get("properties") or {}
-            enabled_config_key = f"{str(plugin_name).upper()}_ENABLED"
-            enabled_prop_schema = properties.get(enabled_config_key)
-            if not isinstance(enabled_prop_schema, dict) or "boolean" not in _schema_types(enabled_prop_schema):
-                enabled_config_key = ""
-            config_fields = [
-                self._build_plugin_config_field(str(plugin_name), str(config_key), prop_schema, runtime_config)
-                for config_key, prop_schema in properties.items()
-                if isinstance(prop_schema, dict)
-            ]
-            cards.append(
-                {
-                    "name": str(plugin_name),
-                    "label": label,
-                    "checked": str(plugin_name) in selected_values,
-                    "checkbox_id": f"id_{field_name}_{index}",
-                    "enabled_config_key": enabled_config_key,
-                    "description": str(schema.get("description") or "").strip(),
-                    "source_url": f"https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/{plugin_name}",
-                    "docs_url": f"https://archivebox.github.io/abx-plugins/#{plugin_name}",
-                    "required_plugins": [str(item) for item in schema.get("required_plugins") or []],
-                    "required_binary_links": _build_required_binary_links(
-                        schema.get("required_binaries") or [],
-                        runtime_config,
-                        binary_url_lookup,
-                    ),
-                    "config_fields": config_fields,
-                    "config_count": len(config_fields),
-                },
-            )
-        return cards
-
-    def _build_plugin_config_field(
-        self,
-        plugin_name: str,
-        config_key: str,
-        prop_schema: Mapping[str, Any],
-        runtime_config: Mapping[str, Any],
-    ) -> dict[str, Any]:
-        schema_types = _schema_types(prop_schema)
-        enum = prop_schema.get("enum")
-        input_name = _plugin_config_input_name(plugin_name, config_key)
-        current_value = runtime_config.get(config_key, prop_schema.get("default", ""))
-        if self.is_bound and input_name in self.data:
-            try:
-                current_value = _coerce_plugin_config_value(self.data.get(input_name), prop_schema)
-            except (TypeError, ValueError, json.JSONDecodeError, forms.ValidationError):
-                current_value = self.data.get(input_name)
-
-        default_value = prop_schema.get("default", "")
-        fallback_key = prop_schema.get("x-fallback")
-        default_display = f"{{{fallback_key}}}" if fallback_key else default_value
-        # A field is sensitive if either the schema explicitly marks it
-        # (``x-sensitive``) or the key name matches our credential heuristic
-        # (``*TOKEN*`` / ``*SECRET*`` / ``*API_KEY*`` / ``*APIKEY*``). The
-        # plugin grid lives on user-facing pages, so we redact the value,
-        # render a password input, and on empty submit we preserve the
-        # previously-saved value in ``clean_plugin_config_overrides`` below.
-        from archivebox.config.common import is_sensitive_config_key
-
-        is_sensitive = bool(prop_schema.get("x-sensitive")) or is_sensitive_config_key(config_key)
-        input_value = "" if is_sensitive else _jsonish(current_value)
-        field_kind = "text"
-        input_type = "text"
-        options = []
-
-        if "boolean" in schema_types:
-            field_kind = "boolean"
-            input_value = "true" if bool(current_value) else "false"
-        elif isinstance(enum, list) and enum:
-            field_kind = "select"
-            options = [
-                {
-                    "value": str(option),
-                    "label": str(option),
-                    "selected": str(option) == str(current_value),
-                }
-                for option in enum
-            ]
-        elif "integer" in schema_types or "number" in schema_types:
-            field_kind = "number"
-            input_type = "number"
-        elif "array" in schema_types or "object" in schema_types:
-            field_kind = "json"
-            input_value = "" if is_sensitive else json.dumps(current_value, indent=2, sort_keys=True, default=str)
-        elif is_sensitive:
-            input_type = "password"
-
-        return {
-            "key": config_key,
-            "input_name": input_name,
-            "kind": field_kind,
-            "input_type": input_type,
-            "value": input_value,
-            "checked": bool(current_value),
-            "options": options,
-            "description": str(prop_schema.get("description") or "").strip(),
-            "default": _jsonish(default_display),
-            "current": "configured" if is_sensitive and current_value else _jsonish(current_value),
-            "current_url": self.plugin_config_binary_urls.get(config_key, "") if str(config_key).endswith("_BINARY") else "",
-            "is_sensitive": is_sensitive,
-            "minimum": prop_schema.get("minimum"),
-            "maximum": prop_schema.get("maximum"),
-            "pattern": prop_schema.get("pattern"),
-            "type_label": " / ".join(schema_types),
-        }
-
-    def clean_plugin_config_overrides(self, effective_config: Mapping[str, Any] | None = None) -> dict[str, Any]:
-        if not self.is_bound:
-            return {}
-
-        effective_config = effective_config or get_config()
-        overrides: dict[str, Any] = {}
-        sources: dict[str, str] = {}
-
-        for plugin_name, schema in discover_plugin_configs().items():
-            for config_key, prop_schema in (schema.get("properties") or {}).items():
-                if not isinstance(prop_schema, dict):
-                    continue
-
-                input_name = _plugin_config_input_name(plugin_name, config_key)
-                if input_name not in self.data:
-                    continue
-
-                raw_value: Any = self.data.get(input_name)
-                if "array" in _schema_types(prop_schema) and isinstance(prop_schema.get("enum"), list):
-                    raw_value = self.data.getlist(input_name)
-
-                from archivebox.config.common import is_sensitive_config_key
-
-                if (prop_schema.get("x-sensitive") or is_sensitive_config_key(config_key)) and raw_value == "":
-                    continue
-
-                try:
-                    coerced_value = _coerce_plugin_config_value(raw_value, prop_schema)
-                except (TypeError, ValueError, json.JSONDecodeError) as err:
-                    self.add_error("config", forms.ValidationError(f"{config_key}: {err}"))
-                    continue
-                except forms.ValidationError as err:
-                    self.add_error("config", forms.ValidationError(f"{config_key}: {err.messages[0]}"))
-                    continue
-
-                base_value = effective_config.get(config_key, prop_schema.get("default", ""))
-                if _same_config_value(coerced_value, base_value):
-                    continue
-
-                existing_value = overrides.get(config_key)
-                if config_key in overrides and not _same_config_value(existing_value, coerced_value):
-                    self.add_error(
-                        "config",
-                        forms.ValidationError(
-                            f"{config_key} was set differently under {sources[config_key]} and {plugin_name}. Set it once in Custom config overrides.",
-                        ),
-                    )
-                    continue
-
-                overrides[config_key] = coerced_value
-                sources[config_key] = plugin_name
-
-        return overrides
-
-    def plugin_config_keys(self) -> set[str]:
-        return {
-            str(config_key)
-            for schema in discover_plugin_configs().values()
-            for config_key, prop_schema in (schema.get("properties") or {}).items()
-            if isinstance(prop_schema, dict)
-        }
-
-
-_BINARY_TEMPLATE_PATTERN = re.compile(r"\{([A-Z_][A-Z0-9_]*)\}")
-
-
-def _resolve_required_binary_name(template_name: str, runtime_config: Mapping[str, Any]) -> str:
-    if "{" not in template_name:
-        return template_name
-
-    def _replace(match: re.Match[str]) -> str:
-        key = match.group(1)
-        try:
-            value = runtime_config.get(key)
-        except Exception:
-            value = None
-        if value is None or value == "":
-            return match.group(0)
-        return str(value)
-
-    resolved = _BINARY_TEMPLATE_PATTERN.sub(_replace, template_name).strip()
-    if not resolved:
-        return template_name
-    return Path(resolved).name if "/" in resolved else resolved
-
-
-def _iter_required_binary_names(
-    required_binaries: Iterable[Any],
-    runtime_config: Mapping[str, Any],
-) -> Iterable[str]:
-    for item in required_binaries or []:
-        if not isinstance(item, dict):
-            continue
-        raw_name = str(item.get("name") or "").strip()
-        if not raw_name:
-            continue
-        resolved = _resolve_required_binary_name(raw_name, runtime_config)
-        if resolved:
-            yield resolved
-
-
-def _build_required_binary_url_lookup(
-    plugin_configs: Mapping[str, dict[str, Any]],
-    runtime_config: Mapping[str, Any],
-) -> dict[str, str]:
-    """Resolve admin URLs for every required binary across all plugin schemas in a single DB query."""
-    from archivebox.config.views import get_environment_binary_url, get_installed_binary_change_url
-    from archivebox.machine.models import Binary, Machine
-
-    resolved_names: set[str] = set()
-    for schema in plugin_configs.values():
-        for name in _iter_required_binary_names(schema.get("required_binaries") or [], runtime_config):
-            resolved_names.add(name)
-
-    if not resolved_names:
-        return {}
-
-    machine = Machine.current()
-    name_to_binary: dict[str, Binary] = {}
-    for binary in (
-        Binary.objects.filter(machine=machine, name__in=resolved_names)
-        .exclude(abspath="")
-        .exclude(abspath__isnull=True)
-        .order_by("-modified_at")
-    ):
-        key = binary.name.lower()
-        if key not in name_to_binary:
-            name_to_binary[key] = binary
-
-    return {
-        name: (get_installed_binary_change_url(name, name_to_binary.get(name.lower())) or get_environment_binary_url(name))
-        for name in resolved_names
-    }
-
-
-def _build_required_binary_links(
-    required_binaries: list[dict[str, Any]],
-    runtime_config: Mapping[str, Any],
-    binary_url_lookup: Mapping[str, str] | None = None,
-) -> list[dict[str, str]]:
-    from archivebox.config.views import get_environment_binary_url
-
-    links: list[dict[str, str]] = []
-    seen: set[str] = set()
-    for resolved in _iter_required_binary_names(required_binaries, runtime_config):
-        if resolved in seen:
-            continue
-        seen.add(resolved)
-        url = (binary_url_lookup or {}).get(resolved) or get_environment_binary_url(resolved)
-        links.append({"name": resolved, "url": url})
-    return links
-
-
-def get_plugin_config_binary_urls(runtime_config: Mapping[str, Any]) -> dict[str, str]:
-    from archivebox.config.views import get_environment_binary_url, get_installed_binary_change_url
-    from archivebox.machine.models import Binary, Machine
-
-    binary_keys = {
-        str(config_key)
-        for schema in discover_plugin_configs().values()
-        for config_key, prop_schema in (schema.get("properties") or {}).items()
-        if isinstance(prop_schema, dict) and str(config_key).endswith("_BINARY")
-    }
-    urls: dict[str, str] = {}
-    machine = Machine.current()
-    for key in binary_keys:
-        value = str(runtime_config.get(key) or "").strip()
-        if not value:
-            continue
-        name = Path(value).name if "/" in value else value
-        binary = Binary.objects.get_valid_binary(value, machine=machine)
-        if binary is None and "/" in value:
-            binary = (
-                Binary.objects.exclude(abspath="")
-                .exclude(abspath__isnull=True)
-                .filter(machine=machine, abspath=value)
-                .order_by("-modified_at")
-                .first()
-            )
-        if binary is None and name != value:
-            binary = Binary.objects.get_valid_binary(name, machine=machine)
-        urls[key] = get_installed_binary_change_url(getattr(binary, "name", name), binary) or get_environment_binary_url(name)
-    return urls
+def _split_strip(value: str, delimiter: str) -> list[str]:
+    return [part.strip() for part in value.split(delimiter) if part.strip()]
+
+
+def parse_tag_string(value: str | None) -> list[str]:
+    """Parse the legacy tag-editing format used by the admin tag field."""
+    if not value:
+        return []
+
+    if "," not in value and '"' not in value:
+        return sorted(set(_split_strip(value, " ")))
+
+    tags: list[str] = []
+    buffer: list[str] = []
+    deferred_chunks: list[str] = []
+    saw_unquoted_comma = False
+    in_quote = False
+    chars = iter(value)
+
+    try:
+        while True:
+            char = next(chars)
+            if char == '"':
+                if buffer:
+                    deferred_chunks.append("".join(buffer))
+                    buffer = []
+                in_quote = True
+                char = next(chars)
+                while char != '"':
+                    buffer.append(char)
+                    char = next(chars)
+                tag = "".join(buffer).strip()
+                if tag:
+                    tags.append(tag)
+                buffer = []
+                in_quote = False
+            else:
+                if not saw_unquoted_comma and char == ",":
+                    saw_unquoted_comma = True
+                buffer.append(char)
+    except StopIteration:
+        if buffer:
+            if in_quote and "," in buffer:
+                saw_unquoted_comma = True
+            deferred_chunks.append("".join(buffer))
+
+    delimiter = "," if saw_unquoted_comma else " "
+    for chunk in deferred_chunks:
+        tags.extend(_split_strip(chunk, delimiter))
+
+    return sorted(set(tags))
+
+
+def edit_string_for_tag_names(tags) -> str:
+    names = []
+    for tag in tags:
+        name = tag.name
+        names.append(f'"{name}"' if "," in name or " " in name else name)
+    return ", ".join(sorted(names))
 
 
 class AddLinkForm(PluginConfigFormMixin, forms.Form):
+    allow_crawl_execution_config_fields = False
+
     # Basic fields
     url = forms.CharField(
         label="URLs",
-        strip=True,
+        strip=False,
         widget=forms.Textarea(
             attrs={
                 "data-url-regex": URL_REGEX.pattern,
                 "placeholder": (
-                    "Enter URLs to archive, as one per line, CSV, JSON, or embedded in text "
-                    "(e.g. markdown, HTML, etc.). Examples:\n"
-                    "https://example.com\n"
-                    "https://news.ycombinator.com,https://news.google.com\n"
-                    "[ArchiveBox](https://github.com/ArchiveBox/ArchiveBox)"
+                    "\n",
+                    "Enter URL(s) to archive. Any format is ok: one per line, CSV, JSON, embedded in text, etc."
+                    "Examples:\n\n"
+                    "https://example.com\n\n"
+                    "https://news.ycombinator.com,https://news.google.com\n\n"
+                    "Or any text-based content [containing URLs](https://github.com/ArchiveBox/ArchiveBox)...",
                 ),
             },
         ),
@@ -819,7 +299,10 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 
     def __init__(self, *args, **kwargs):
         self.request = kwargs.pop("request", None)
-        self.can_override_crawl_config = bool(self.request and is_admin_user(self.request))
+        self.can_override_crawl_config = False
+        if self.request is not None:
+            user = self.request.user
+            self.can_override_crawl_config = bool(user.is_authenticated and user.is_active and user.is_superuser)
         super().__init__(*args, **kwargs)
 
         default_persona = Persona.get_or_create_default()
@@ -895,12 +378,9 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 
     def clean_url(self):
         value = self.cleaned_data.get("url") or ""
-        valid_urls = []
-        for url in find_all_urls(value):
-            valid_urls.append(url)
-        if not valid_urls:
+        if not list(find_all_urls(value)):
             raise forms.ValidationError("Enter at least one valid URL.")
-        return "\n".join(valid_urls)
+        return value
 
     def clean_url_filters(self):
         from archivebox.crawls.models import Crawl
@@ -1006,7 +486,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 class TagWidget(forms.TextInput):
     def format_value(self, value):
         if value is not None and not isinstance(value, str):
-            value = edit_string_for_tags(value)
+            value = edit_string_for_tag_names(value)
         return super().format_value(value)
 
 
@@ -1015,12 +495,7 @@ class TagField(forms.CharField):
 
     def clean(self, value):
         value = super().clean(value)
-        try:
-            return parse_tags(value)
-        except ValueError:
-            raise forms.ValidationError(
-                "Please provide a comma-separated list of tags.",
-            )
+        return parse_tag_string(value)
 
     def has_changed(self, initial, data):
         # Always return False if the field is disabled since self.bound_data

archivebox/core/middleware.py

@@ -3,6 +3,7 @@ __package__ = "archivebox.core"
 import ipaddress
 import re
 from pathlib import Path
+from django.conf import settings
 from django.utils import timezone
 from django.contrib.auth.middleware import RemoteUserMiddleware
 from django.contrib.auth.models import AnonymousUser
@@ -15,7 +16,7 @@ from django.http import HttpResponseForbidden, HttpResponseNotModified
 from archivebox.config.common import get_config
 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH
-from archivebox.core.host_util import (
+from archivebox.core.routes_util import (
     build_snapshot_url,
     build_admin_url,
     build_web_url,
@@ -24,7 +25,6 @@ from archivebox.core.host_util import (
     get_base_host,
     get_listen_host,
     get_listen_subdomain,
-    get_public_host,
     get_web_host,
     host_matches,
     is_snapshot_subdomain,
@@ -42,7 +42,7 @@ def _admin_login_hint_cookie_domain(config) -> str | None:
     NOTE: this cookie carries only the single bit "user is logged in on
     admin somewhere"; it MUST NOT be confused with the session cookie,
     which stays admin-host-scoped (see core/settings.py
-    SESSION_COOKIE_DOMAIN comment — admin/public is a security boundary).
+    SESSION_COOKIE_DOMAIN comment — admin/web is a security boundary).
 
     Returns the hostname portion of ``get_base_host`` (which respects
     ``BASE_URL`` and falls back to the local-bind mapping). Strips the
@@ -85,6 +85,34 @@ def TimezoneMiddleware(get_response):
     return middleware
 
 
+def AdminCookieIsolationMiddleware(get_response):
+    def middleware(request):
+        response = get_response(request)
+
+        config = request.__dict__.get("archivebox_config")
+        if config is None:
+            config = get_config(resolve_plugins=False)
+            request.archivebox_config = config
+        if not config.USES_SUBDOMAIN_ROUTING:
+            return response
+
+        request_host = (request.get_host() or "").lower()
+        if host_matches(request_host, get_admin_host(config=config)):
+            return response
+
+        if host_matches(request_host, get_web_host(config=config)):
+            for cookie_name in tuple(response.cookies.keys()):
+                if cookie_name != ADMIN_LOGIN_HINT_COOKIE:
+                    response.cookies.pop(cookie_name, None)
+            return response
+
+        response.cookies.pop(settings.SESSION_COOKIE_NAME, None)
+        response.cookies.pop(settings.CSRF_COOKIE_NAME, None)
+        return response
+
+    return middleware
+
+
 def CacheControlMiddleware(get_response):
     snapshot_path_re = re.compile(r"^/[^/]+/\\d{8}/[^/]+/[0-9a-fA-F-]{8,36}/")
     static_cache_key = (get_COMMIT_HASH() or VERSION or "dev").strip()
@@ -119,7 +147,7 @@ def CacheControlMiddleware(get_response):
 
         if "/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path):
             if not response.get("Cache-Control"):
-                config = getattr(request, "archivebox_config", None)
+                config = request.__dict__.get("archivebox_config")
                 if config is None:
                     config = get_config(resolve_plugins=False)
                     request.archivebox_config = config
@@ -136,7 +164,7 @@ def ServerSecurityModeMiddleware(get_response):
     allowed_methods = {"GET", "HEAD", "OPTIONS"}
 
     def middleware(request):
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
         if config is None:
             config = get_config(resolve_plugins=False)
             request.archivebox_config = config
@@ -169,14 +197,13 @@ def HostRoutingMiddleware(get_response):
             return get_response(request)
 
         request_host = (request.get_host() or "").lower()
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
         if config is None:
             config = get_config(resolve_plugins=False)
             request.archivebox_config = config
         admin_host = get_admin_host(config=config)
         web_host = get_web_host(config=config)
         api_host = get_api_host(config=config)
-        public_host = get_public_host(config=config)
         listen_host = get_listen_host(config=config)
         subdomain = get_listen_subdomain(request_host, config=config)
 
@@ -185,25 +212,7 @@ def HostRoutingMiddleware(get_response):
         if request.path.startswith("/static/") or request.path in {"/favicon.ico", "/robots.txt"}:
             return get_response(request)
 
-        # In subdomain mode with no explicit BASE_URL we can't safely emit
-        # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
-        # request's own Host (via the request-host fallback in get_base_url),
-        # so prepending ``admin.`` to whatever the client sent produces a
-        # redirect chain of ``admin.admin.admin.<host>``. Pass the request
-        # through; the misconfig banner on the rendered page tells the user
-        # to pin BASE_URL so the redirects can resume.
-        if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
-            return get_response(request)
-
-        if config.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host):
-            # ``/add`` is admin-only unless ``PUBLIC_ADD_VIEW`` is on. Without
-            # this redirect, hitting it on public.* falls into AddView's
-            # auth check, bounces through ``/accounts/login/?next=/add/`` →
-            # ``/admin/login/?next=/add/``, and Django admin's LoginView
-            # silently drops ``next`` when the user already has an admin
-            # session — dumping the user on the admin homepage instead of
-            # the add form. Routing the request to admin.* directly lets
-            # AddView run on the host where the session lives.
+        if config.USES_SUBDOMAIN_ROUTING and config.BASE_URL and not host_matches(request_host, admin_host):
             add_should_redirect = not config.PUBLIC_ADD_VIEW and (request.path == "/add" or request.path.startswith("/add/"))
             if (
                 request.path == "/admin"
@@ -217,6 +226,20 @@ def HostRoutingMiddleware(get_response):
                     target = f"{target}?{request.META['QUERY_STRING']}"
                 return redirect(target)
 
+        if subdomain and is_snapshot_subdomain(subdomain):
+            view = SnapshotHostView.as_view()
+            return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
+
+        # In subdomain mode with no explicit BASE_URL we can't safely emit
+        # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
+        # request's own Host (via the request-host fallback in get_base_url),
+        # so prepending ``admin.`` to whatever the client sent produces a
+        # redirect chain of ``admin.admin.admin.<host>``. Pass the request
+        # through; the misconfig banner on the rendered page tells the user
+        # to pin BASE_URL so the redirects can resume.
+        if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
+            return get_response(request)
+
         if not config.USES_SUBDOMAIN_ROUTING:
             if host_matches(request_host, listen_host):
                 return get_response(request)
@@ -282,16 +305,7 @@ def HostRoutingMiddleware(get_response):
             request._cached_user = request.user
             return get_response(request)
 
-        if host_matches(request_host, public_host):
-            if request.COOKIES.get(ADMIN_LOGIN_HINT_COOKIE) == "1" and (request.path == "/public" or request.path.startswith("/public/")):
-                target = build_admin_url("/admin/core/snapshot/", request=request)
-                return redirect(target)
-            return get_response(request)
-
         if subdomain:
-            if is_snapshot_subdomain(subdomain):
-                view = SnapshotHostView.as_view()
-                return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
             view = OriginalDomainHostView.as_view()
             return view(request, domain=subdomain, path=request.path.lstrip("/"))
 
@@ -323,7 +337,7 @@ class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
     header = "HTTP_REMOTE_USER"
 
     def process_request(self, request):
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
         if config is None:
             config = get_config(resolve_plugins=False)
             request.archivebox_config = config

archivebox/core/migrations/0047_archiveresult_status_snapshot_index.py

@@ -0,0 +1,28 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0046_repair_snapshot_permissions"),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunSQL(
+                    sql="CREATE INDEX IF NOT EXISTS archiveresult_status_snap_idx ON core_archiveresult (status, snapshot_id)",
+                    reverse_sql="DROP INDEX IF EXISTS archiveresult_status_snap_idx",
+                ),
+            ],
+            state_operations=[
+                migrations.AddIndex(
+                    model_name="archiveresult",
+                    index=models.Index(fields=["status", "snapshot"], name="archiveresult_status_snap_idx"),
+                ),
+            ],
+        ),
+        migrations.RunSQL(
+            sql="ANALYZE core_archiveresult",
+            reverse_sql=migrations.RunSQL.noop,
+        ),
+    ]

archivebox/core/migrations/0048_remove_archiveresult_config.py

@@ -0,0 +1,14 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0047_archiveresult_status_snapshot_index"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="archiveresult",
+            name="config",
+        ),
+    ]

archivebox/core/migrations/0049_alter_snapshot_url.py

@@ -0,0 +1,17 @@
+# Generated by Django 6.0.6 on 2026-06-04 22:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0048_remove_archiveresult_config"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="url",
+            field=models.TextField(db_index=True),
+        ),
+    ]

archivebox/core/migrations/0050_snapshot_permissions_not_null.py

@@ -0,0 +1,69 @@
+import django.db.models.fields.json
+import django.db.models.functions.comparison
+from django.db import migrations, models
+from django.db.models import Q
+
+
+VALID_PERMISSIONS = {"public", "unlisted", "private"}
+BATCH_SIZE = 1000
+
+
+def normalize_permissions(value, default="private"):
+    value = str(value or "").strip().lower()
+    return value if value in VALID_PERMISSIONS else default
+
+
+def hydrate_snapshot_permissions(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    db_alias = schema_editor.connection.alias
+    missing_permissions = Q(permissions__isnull=True) | (Q(permissions__isnull=False) & ~Q(permissions__in=VALID_PERMISSIONS))
+    batch = []
+
+    snapshots = (
+        Snapshot.objects.using(db_alias)
+        .filter(missing_permissions)
+        .select_related("crawl")
+        .only("id", "config", "crawl__permissions")
+        .iterator(chunk_size=BATCH_SIZE)
+    )
+    for snapshot in snapshots:
+        config = dict(snapshot.config or {})
+        config["PERMISSIONS"] = normalize_permissions(snapshot.crawl.permissions)
+        snapshot.config = config
+        batch.append(snapshot)
+        if len(batch) >= BATCH_SIZE:
+            Snapshot.objects.using(db_alias).bulk_update(batch, ["config"], batch_size=BATCH_SIZE)
+            batch.clear()
+
+    if batch:
+        Snapshot.objects.using(db_alias).bulk_update(batch, ["config"], batch_size=BATCH_SIZE)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("crawls", "0016_hydrate_crawl_permissions"),
+        ("core", "0049_alter_snapshot_url"),
+    ]
+
+    operations = [
+        migrations.RunPython(hydrate_snapshot_permissions, migrations.RunPython.noop),
+        migrations.RemoveField(
+            model_name="snapshot",
+            name="permissions",
+        ),
+        migrations.AddField(
+            model_name="snapshot",
+            name="permissions",
+            field=models.GeneratedField(
+                db_index=True,
+                db_persist=True,
+                editable=False,
+                expression=django.db.models.functions.comparison.Coalesce(
+                    django.db.models.fields.json.KeyTextTransform("PERMISSIONS", "config"),
+                    models.Value("private"),
+                    output_field=models.CharField(max_length=16),
+                ),
+                output_field=models.CharField(max_length=16),
+            ),
+        ),
+    ]

archivebox/core/permissions.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from django.db.models import Q, QuerySet
+from django.db.models import QuerySet
 from django.http import HttpRequest
 
 PERMISSIONS_PUBLIC = "public"
@@ -32,10 +32,7 @@ def is_admin_user(request: HttpRequest) -> bool:
 
 
 def get_snapshot_permissions(snapshot) -> str:
-    permissions = snapshot.permissions
-    if not permissions:
-        permissions = snapshot.crawl.permissions
-    return normalize_permissions(permissions)
+    return normalize_permissions(snapshot.permissions)
 
 
 def can_view_snapshot(request: HttpRequest, snapshot) -> bool:
@@ -56,10 +53,7 @@ def filter_personas_by_permissions(queryset: QuerySet, allowed_permissions: set[
 def filter_snapshots_by_permissions(queryset: QuerySet, *, direct: bool = False, allowed_permissions: set[str] | None = None) -> QuerySet:
     allowed_permissions = allowed_permissions or ({PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED} if direct else {PERMISSIONS_PUBLIC})
     allowed = sorted(allowed_permissions)
-    return queryset.exclude(
-        (Q(permissions__isnull=False) & ~Q(permissions__in=allowed))
-        | (Q(permissions__isnull=True) & (Q(crawl__permissions__isnull=True) | ~Q(crawl__permissions__in=allowed))),
-    )
+    return queryset.filter(permissions__in=allowed)
 
 
 def public_snapshots_queryset(queryset: QuerySet) -> QuerySet:

archivebox/core/preview_util.py

@@ -0,0 +1,4 @@
+__package__ = "archivebox.core"
+
+
+EXTENSION_SCREENSHOT_PLUGIN = "chrome_extension_screenshot"

archivebox/core/recovery_util.py

@@ -1,26 +1,39 @@
 from __future__ import annotations
 
+from pathlib import Path
+
 from django.utils import timezone
 from rich.console import Console
 
 
-def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int]:
+def _is_signal_interrupted_exit(exit_code: int | None) -> bool:
+    return exit_code is not None and (exit_code < 0 or exit_code >= 128)
+
+
+def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | None = None) -> dict[str, int]:
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.services.archive_result_service import _collect_output_metadata
     from archivebox.machine.models import Process
+    from django.core.exceptions import ValidationError
     from django.db.models import Exists, OuterRef, Q, Subquery, Value
     from django.db.models.functions import Coalesce
 
     now = timezone.now()
     recovery_console = Console(stderr=True, highlight=False, soft_wrap=True)
+    crawl_filter = {"id": crawl_id} if crawl_id else {}
+    snapshot_filter = {"crawl_id": crawl_id} if crawl_id else {}
+    result_filter = {"snapshot__crawl_id": crawl_id} if crawl_id else {}
     cleaned = {
-        "processes_stale_running": Process.cleanup_stale_running(),
-        "processes_orphaned_workers": Process.cleanup_orphaned_workers(),
-        "chrome_processes_orphaned": Process.cleanup_orphaned_chrome() if include_chrome else 0,
+        "processes_stale_running": 0 if crawl_id else Process.cleanup_stale_running(),
+        "processes_orphaned_workers": 0 if crawl_id else Process.cleanup_orphaned_workers(),
+        "chrome_processes_orphaned": Process.cleanup_orphaned_chrome() if include_chrome and not crawl_id else 0,
         "crawls_queued_without_retry_at": 0,
         "snapshots_queued_without_retry_at": 0,
         "archiveresults_backoff": 0,
+        "snapshots_queued_plugin_rows_waiting_on_stale_lease": 0,
         "archiveresults_started_without_running_process": 0,
+        "archiveresults_missing_for_orphaned_hook_processes": 0,
         "snapshots_started_without_running_results": 0,
         "crawls_started_with_due_snapshots": 0,
         "crawls_started_waiting_on_future_snapshots": 0,
@@ -34,7 +47,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
     )
     active_child_snapshots = Snapshot.objects.filter(
         crawl_id=OuterRef("pk"),
-        status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.PAUSED],
+        status__in=Snapshot.OPEN_STATES,
     )
     due_child_snapshots = active_child_snapshots.exclude(status=Snapshot.StatusChoices.PAUSED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__lte=now),
@@ -48,14 +61,16 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
     cleaned["crawls_queued_without_retry_at"] = Crawl.objects.filter(
         status=Crawl.StatusChoices.QUEUED,
         retry_at__isnull=True,
+        **crawl_filter,
     ).update(retry_at=now, modified_at=now)
     cleaned["snapshots_queued_without_retry_at"] = Snapshot.objects.filter(
         status=Snapshot.StatusChoices.QUEUED,
         retry_at__isnull=True,
-        crawl__status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED],
+        crawl__status__in=Crawl.RUNNABLE_STATES,
+        **snapshot_filter,
     ).update(retry_at=now, modified_at=now)
-    backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF)
-    orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED).exclude(
+    backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF, **result_filter)
+    orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED, **result_filter).exclude(
         process__status=Process.StatusChoices.RUNNING,
     )
     # ArchiveResult has no retry_at scheduler. Wake only the parent Snapshots
@@ -73,6 +88,24 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
         retry_at__isnull=True,
     ).update(retry_at=now, modified_at=now)
     cleaned["archiveresults_backoff"] = backoff_results.update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now)
+    # Targeted plugin rows on final/paused Snapshots are scheduled through the
+    # parent Snapshot.retry_at. retry_at=NULL is the normal idle marker for a
+    # sealed Snapshot and must not be interpreted as queued work just because
+    # old/synthetic ArchiveResult rows exist. If takeover kills the runner
+    # after it leases the Snapshot but before queued ArchiveResult rows finish,
+    # the rows remain QUEUED while retry_at sits in the future. Recovery runs
+    # only after this runner has won the single-runner gate, so it can safely
+    # unlock those stale plugin leases for immediate processing instead of
+    # waiting out the previous owner's full lock timeout.
+    queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED, **result_filter)
+    cleaned["snapshots_queued_plugin_rows_waiting_on_stale_lease"] = (
+        Snapshot.objects.filter(
+            id__in=queued_plugin_results.values("snapshot_id"),
+            status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
+        )
+        .filter(retry_at__gt=now)
+        .update(retry_at=now, modified_at=now)
+    )
     # Impossible state repair: STARTED ArchiveResults without a live Process
     # have no owner left to emit completion. Requeue only the result row; the
     # snapshot/crawl schedulers will pick up normal retry processing.
@@ -81,8 +114,96 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
         process=None,
         modified_at=now,
     )
+    orphaned_hook_processes = Process.objects.filter(
+        process_type=Process.TypeChoices.HOOK,
+        archiveresult__isnull=True,
+    ).exclude(status=Process.StatusChoices.RUNNING)
+    crawl_snapshot_ids = (
+        {str(snapshot_id) for snapshot_id in Snapshot.objects.filter(crawl_id=crawl_id).values_list("id", flat=True)} if crawl_id else None
+    )
+    if crawl_snapshot_ids is not None:
+        # Targeted `archivebox run --crawl-id ...` is used by foreground add/update
+        # commands and by resume of one existing crawl. It must still repair bad
+        # hook state for that crawl, but it must not scan historical orphaned hook
+        # rows from unrelated crawls before the first snapshot can run.
+        if crawl_snapshot_ids:
+            snapshot_pwd_filter = Q()
+            for snapshot_id in crawl_snapshot_ids:
+                snapshot_pwd_filter |= Q(pwd__contains=snapshot_id)
+            orphaned_hook_processes = orphaned_hook_processes.filter(snapshot_pwd_filter)
+        else:
+            orphaned_hook_processes = orphaned_hook_processes.none()
+    for process in orphaned_hook_processes.only("id", "pwd", "cmd", "process_type", "status"):
+        hook_script_name = process.hook_script_name
+        if not hook_script_name or not process.pwd:
+            continue
+        plugin_dir = Path(process.pwd)
+        if crawl_snapshot_ids is not None and plugin_dir.parent.name not in crawl_snapshot_ids:
+            continue
+        try:
+            # Old or synthetic hook Process rows can point at arbitrary paths.
+            # Only paths whose parent directory is a valid Snapshot id can be
+            # reconstructed into ArchiveResult rows.
+            snapshot = Snapshot.objects.filter(id=plugin_dir.parent.name, **snapshot_filter).first()
+        except ValidationError:
+            continue
+        if snapshot is None:
+            continue
+        result, created = ArchiveResult.objects.get_or_create(
+            snapshot=snapshot,
+            plugin=plugin_dir.name,
+            hook_name=Path(hook_script_name).stem,
+            defaults={
+                "status": ArchiveResult.StatusChoices.QUEUED,
+            },
+        )
+        if result.status == ArchiveResult.StatusChoices.QUEUED:
+            requeue_snapshot = False
+            # A runner can die after the hook Process exits but before the
+            # ProcessCompletedEvent projector links/finalizes ArchiveResult.
+            # Reconstruct only that exact hook row from the durable Process row.
+            output_files, output_size, output_mimetypes = _collect_output_metadata(plugin_dir)
+            result.process = process
+            if _is_signal_interrupted_exit(process.exit_code):
+                # The owning runner died or was asked to stop while the hook was
+                # still active. Keep the work item queued so takeover retries the
+                # same hook; treating an unknown signal exit as success would
+                # silently skip unfinished side effects.
+                result.output_files = {}
+                result.output_size = 0
+                result.output_mimetypes = ""
+                result.output_str = ""
+                result.status = ArchiveResult.StatusChoices.QUEUED
+                requeue_snapshot = True
+            else:
+                result.output_files = output_files
+                result.output_size = output_size
+                result.output_mimetypes = output_mimetypes
+                result.output_str = process.stderr if process.exit_code not in (0, None) else ""
+                result.status = (
+                    ArchiveResult.StatusChoices.FAILED
+                    if process.exit_code not in (0, None)
+                    else (ArchiveResult.StatusChoices.SUCCEEDED if output_files else ArchiveResult.StatusChoices.NORESULTS)
+                )
+            result.save(
+                update_fields=[
+                    "process",
+                    "output_files",
+                    "output_size",
+                    "output_mimetypes",
+                    "output_str",
+                    "status",
+                    "modified_at",
+                ],
+            )
+            if requeue_snapshot:
+                Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
+        if created:
+            cleaned["archiveresults_missing_for_orphaned_hook_processes"] += 1
+            Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
     started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+        **snapshot_filter,
     )
 
     # Broken lock repair: STARTED + retry_at=NULL or retry_at in the future
@@ -107,6 +228,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
     # are already final.
     recoverable_started_crawls = Crawl.objects.filter(status=Crawl.StatusChoices.STARTED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+        **crawl_filter,
     )
 
     due_started_crawls = recoverable_started_crawls.annotate(has_due_child=Exists(due_child_snapshots)).filter(has_due_child=True)

archivebox/core/routes_util.py

@@ -4,12 +4,14 @@ import re
 from typing import Any
 from urllib.parse import urlparse
 
+from django.utils.http import url_has_allowed_host_and_scheme
+
 from archivebox.config.common import get_config
 
 
 _SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
 _SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P<suffix>[0-9a-fA-F]{12})$")
-_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api", "public")
+_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api")
 
 
 def split_host_port(host: str) -> tuple[str, str | None]:
@@ -102,7 +104,7 @@ def _with_port(host: str, port: str | None) -> str:
 
 
 def strip_role_subdomain(host: str) -> str:
-    """Strip leading ``admin.`` / ``web.`` / ``api.`` / ``public.`` / ``snap-*.``
+    """Strip leading ``admin.`` / ``web.`` / ``api.`` / ``snap-*.``
     labels from a host (preserving the port). Strips repeatedly so an
     already-compounded host like ``snap-X.snap-X.<base>`` reduces all the
     way down to ``<base>``.
@@ -172,13 +174,12 @@ def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kw
             return f"{scheme}://{_with_port('archivebox.localhost', req_port)}"
         # C) Per-request fallback: when ``BASE_URL`` is unset and CSRF didn't
         # give us a single origin, trust the request's Host header — but first
-        # peel off any ``admin.`` / ``web.`` / ``api.`` / ``public.`` /
-        # ``snap-*.`` label. Otherwise the URL builders below prepend their own
-        # role label onto a host that already carries one, producing the
-        # ``snap-X.snap-X.snap-X.<base>`` compounding bug. Django has already
-        # admitted the host via ALLOWED_HOSTS; the misconfig banner surfaces
-        # the case where the resulting URL doesn't match what the operator
-        # probably intended.
+        # peel off any ``admin.`` / ``web.`` / ``api.`` / ``snap-*.`` label.
+        # Otherwise the URL builders below prepend their own role label onto a
+        # host that already carries one, producing the ``snap-X.snap-X.snap-X``
+        # compounding bug. Django has already admitted the host via
+        # ALLOWED_HOSTS; the misconfig banner surfaces the case where the
+        # resulting URL doesn't match what the operator probably intended.
         canonical_host = strip_role_subdomain(request.get_host())
         return f"{scheme}://{canonical_host}"
 
@@ -220,13 +221,6 @@ def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) ->
     return _build_base_host("api", config=config)
 
 
-def get_public_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_host(config=config)
-    return _build_base_host("public", config=config)
-
-
 def get_snapshot_subdomain(snapshot_id: str) -> str:
     normalized = re.sub(r"[^0-9a-fA-F]", "", snapshot_id or "")
     suffix = (normalized[-12:] if len(normalized) >= 12 else normalized).lower()
@@ -257,6 +251,8 @@ def get_snapshot_lookup_key(snapshot_ref: str) -> str:
     match = _SNAPSHOT_SUBDOMAIN_RE.match(value)
     if match:
         return match.group("suffix")
+    if _SNAPSHOT_ID_RE.match(value):
+        return re.sub(r"[^0-9a-fA-F]", "", value).lower()
     return value
 
 
@@ -290,6 +286,54 @@ def host_matches(request_host: str, target_host: str) -> bool:
     return True
 
 
+def is_allowed_archivebox_redirect_url(url: str | None, request=None, config: dict[str, Any] | None = None) -> bool:
+    """Allow login redirects to this host or to the explicit ArchiveBox BASE_URL family.
+
+    Django already rejects arbitrary absolute ``next=`` URLs for the current host.
+    ArchiveBox also has legitimate cross-subdomain handoffs (admin/web/api/snap-*),
+    but only when BASE_URL is pinned. Without an explicit BASE_URL we do not widen
+    trust based on a request Host header that may be user supplied.
+    """
+    if not url:
+        return False
+
+    require_https = bool(request and request.is_secure())
+    request_host = request.get_host() if request else ""
+    if url_has_allowed_host_and_scheme(url=url, allowed_hosts={request_host}, require_https=require_https):
+        return True
+
+    parsed = urlparse(url)
+    if not parsed.netloc:
+        return False
+    if parsed.scheme not in {"http", "https"}:
+        return False
+    if require_https and parsed.scheme != "https":
+        return False
+    if parsed.username or parsed.password:
+        return False
+
+    config = config or get_config()
+    base_url = _normalize_base_url(config.BASE_URL)
+    if not base_url:
+        return False
+
+    base_host, base_port = split_host_port(urlparse(base_url).netloc)
+    if not base_host:
+        return False
+
+    redirect_host = (parsed.hostname or "").lower()
+    if redirect_host != base_host and not redirect_host.endswith(f".{base_host}"):
+        return False
+
+    try:
+        redirect_port = str(parsed.port) if parsed.port else None
+    except ValueError:
+        return False
+    if base_port:
+        return redirect_port == base_port
+    return redirect_port is None
+
+
 def _scheme_from_request(request=None, config: dict[str, Any] | None = None) -> str:
     config = config or get_config()
     override = _normalize_base_url(config.BASE_URL)
@@ -328,13 +372,6 @@ def get_api_base_url(request=None, config: dict[str, Any] | None = None, **confi
     return _build_base_url_for_host(_build_base_host("api", request=request, config=config), request=request, config=config)
 
 
-def get_public_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_url(request=request, config=config)
-    return _build_base_url_for_host(_build_base_host("public", request=request, config=config), request=request, config=config)
-
-
 def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     if not config.USES_SUBDOMAIN_ROUTING:

archivebox/core/settings.py

@@ -13,7 +13,7 @@ import archivebox
 
 from archivebox.config.constants import CONSTANTS
 from archivebox.config.common import get_config
-from archivebox.core.host_util import normalize_base_url, get_admin_base_url, get_api_base_url
+from archivebox.core.routes_util import get_api_base_url, get_admin_base_url, get_base_url, normalize_base_url
 from .settings_logging import SETTINGS_LOGGING
 
 
@@ -63,23 +63,28 @@ INSTALLED_APPS = [
     # Our ArchiveBox-provided apps (use fully qualified names)
     # NOTE: Order matters! Apps with migrations that depend on other apps must come AFTER their dependencies
     # "archivebox.config",  # ArchiveBox config settings (no models, not a real Django app)
+    "archivebox.plugins",  # plugin discovery, hook helpers, config UI, and plugin metadata views
+    "archivebox.search",  # search backend query helpers, admin search UI, and daemon integrations
     "archivebox.machine",  # handles collecting and storing information about the host machine, network interfaces, binaries, etc.
     "archivebox.workers",  # handles starting and managing background workers and processes (orchestrators and actors)
     "archivebox.personas",  # handles Persona and session management
     "archivebox.core",  # core django model with Snapshot, ArchiveResult, etc. (crawls depends on this)
     "archivebox.crawls",  # handles Crawl and CrawlSchedule models and management (depends on core)
+    "archivebox.progressmonitor",  # live progress endpoint and admin monitor template
     "archivebox.api",  # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
-    # ArchiveBox plugins (hook-based plugins no longer add Django apps)
-    # Use hooks.py discover_hooks() for plugin functionality
+    "abx_plugins.plugins.opencode",
     # 3rd-party apps from PyPI that need to be loaded last
     "admin_data_views",  # handles rendering some convenient automatic read-only views of data in Django admin
     "django_extensions",  # provides Django Debug Toolbar (and other non-debug helpers)
 ]
 
+DJANGO_OBJECT_ACTIONS_DEFAULT_HTTP_METHOD = "POST"
+
 
 MIDDLEWARE = [
     "archivebox.core.middleware.TimezoneMiddleware",
     "django.middleware.security.SecurityMiddleware",
+    "archivebox.core.middleware.AdminCookieIsolationMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.common.CommonMiddleware",
     "archivebox.api.middleware.ApiCorsMiddleware",
@@ -173,9 +178,9 @@ except ImportError:
 
 STATIC_URL = "/static/"
 TEMPLATES_DIR_NAME = "templates"
-CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONFIG.CUSTOM_TEMPLATES_DIR) and os.access(CONFIG.CUSTOM_TEMPLATES_DIR, os.R_OK)
+CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK)
 STATICFILES_DIRS = [
-    *([str(CONFIG.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
+    *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
     # *[
     #     str(plugin_dir / 'static')
     #     for plugin_dir in PLUGIN_DIRS.values()
@@ -186,7 +191,7 @@ STATICFILES_DIRS = [
 ]
 
 TEMPLATE_DIRS = [
-    *([str(CONFIG.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
+    *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
     # *[
     #     str(plugin_dir / 'templates')
     #     for plugin_dir in PLUGIN_DIRS.values()
@@ -275,7 +280,7 @@ MIGRATION_MODULES = {"signal_webhooks": None}
 
 # Django requires DEFAULT_AUTO_FIELD to subclass AutoField (BigAutoField, SmallAutoField, etc.)
 # Cannot use UUIDField here until Django 6.0 introduces DEFAULT_PK_FIELD setting
-# For now: manually add `id = models.UUIDField(primary_key=True, default=uuid7, ...)` to all models
+# For now: manually add `id = CompactUUIDField(primary_key=True, default=uuid7, ...)` to all models
 # OR inherit from ModelWithUUID base class which provides UUID primary key
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 
@@ -333,7 +338,7 @@ STORAGES = {
         "BACKEND": "django.core.files.storage.FileSystemStorage",
         "OPTIONS": {
             "base_url": "/archive/",
-            "location": CONFIG.ARCHIVE_DIR,
+            "location": CONSTANTS.ARCHIVE_DIR,
         },
     },
     # "snapshots": {
@@ -415,11 +420,28 @@ SECURE_BROWSER_XSS_FILTER = True
 SECURE_CONTENT_TYPE_NOSNIFF = True
 SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
 
-CSRF_COOKIE_SECURE = False
-SESSION_COOKIE_SECURE = False
+# When BASE_URL is an https:// URL the deployment is HTTPS end-to-end, typically
+# behind a TLS-terminating proxy/tunnel (the bundled traefik/cloudflared profiles,
+# or your own caddy/traefik/nginx) where the proxy -> archivebox hop is plain HTTP, so
+# request.is_secure() / request.scheme would otherwise report http. Honour the
+# proxy's X-Forwarded-Proto so request-derived schemes are correct, and mark the
+# admin session + CSRF cookies Secure so auth cookies are never sent in cleartext.
+# Derived from the RESOLVED base URL's scheme — no separate flag to keep in sync.
+# get_base_url() also covers deployments that only set CSRF_TRUSTED_ORIGINS (the
+# implicit-BASE_URL fallback used on 0.7.x->0.9.x upgrades), so HTTPS hardening
+# isn't lost until BASE_URL is migrated. A plain-http base (e.g. local
+# http://archivebox.localhost:8000) keeps the defaults below.
+BASE_URL_IS_HTTPS = get_base_url(config=CONFIG).strip().lower().startswith("https://")
+if BASE_URL_IS_HTTPS:
+    SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+CSRF_COOKIE_SECURE = BASE_URL_IS_HTTPS
+SESSION_COOKIE_SECURE = BASE_URL_IS_HTTPS
 SESSION_COOKIE_HTTPONLY = True
+SESSION_COOKIE_NAME = f"archivebox_sessionid_{CONSTANTS.COLLECTION_ID}"
+CSRF_COOKIE_NAME = f"archivebox_csrftoken_{CONSTANTS.COLLECTION_ID}"
 # Auth cookies are intentionally scoped to the exact host that set them so
-# the admin session is NOT readable from public.* / web.* / api.* — that
+# the admin session is NOT readable from web.* / api.* — that
 # split is a security boundary, not a UX choice. Subdomains that need to
 # render an "is the user logged in?" indicator must use the single-bit
 # `archivebox_admin_logged_in` hint cookie set by core/middleware.py
@@ -530,11 +552,11 @@ ADMIN_DATA_VIEWS = {
         },
         {
             "route": "plugins/",
-            "view": "archivebox.config.views.plugins_list_view",
+            "view": "archivebox.plugins.views.plugins_list_view",
             "name": "Plugins",
             "items": {
                 "route": "<str:key>/",
-                "view": "archivebox.config.views.plugin_detail_view",
+                "view": "archivebox.plugins.views.plugin_detail_view",
                 "name": "plugin",
             },
         },

archivebox/core/settings_logging.py

@@ -7,6 +7,7 @@ import logging
 
 
 from archivebox.config import CONSTANTS
+from archivebox.misc.logging import STDERR
 
 
 IGNORABLE_URL_PATTERNS = [
@@ -181,7 +182,9 @@ SETTINGS_LOGGING = {
             "formatter": "rich",
             "level": "DEBUG",
             "markup": False,
+            "enable_link_path": False,
             "rich_tracebacks": False,  # Use standard Python tracebacks (no frame/box)
+            "console": STDERR,
             "filters": ["noisyrequestsfilter", "daphneclosetimeout", "asynciocancelledshield", "stripansi"],
         },
         "logfile": {
@@ -196,6 +199,7 @@ SETTINGS_LOGGING = {
         "outbound_webhooks": {
             "class": "rich.logging.RichHandler",
             "markup": False,
+            "enable_link_path": False,
             "rich_tracebacks": False,  # Use standard Python tracebacks (no frame/box)
             "formatter": "outbound_webhooks",
         },

archivebox/core/shutdown_util.py

@@ -134,7 +134,10 @@ def foreground_shutdown_signals(
         # signal for hard foreground-command shutdown. Server/update/run and
         # other non-interactive commands raise immediately so their finally
         # blocks can stop owned children without prompting.
-        if raise_on_first_signal or already_requested:
+        if already_requested:
+            os.write(sys.stdout.fileno(), f"\n[🛑] Got {sig.name} again, exiting immediately.\n".encode())
+            os._exit(130)
+        if raise_on_first_signal:
             raise KeyboardInterrupt
 
     try:
@@ -143,6 +146,8 @@ def foreground_shutdown_signals(
             signal.signal(sig, raise_keyboard_interrupt)
         yield state
     finally:
+        if state.signal_name and previous_active_state is not None and not previous_active_state.signal_name:
+            previous_active_state.signal_name = state.signal_name
         _active_shutdown_state = previous_active_state
         for sig, previous_handler in previous_handlers.items():
             signal.signal(sig, previous_handler)

archivebox/core/snapshot_status.py

@@ -0,0 +1,25 @@
+__package__ = "archivebox.core"
+
+from django.db.models import QuerySet
+
+
+def snapshot_status_values() -> tuple[str, ...]:
+    from archivebox.core.models import Snapshot
+
+    return tuple(Snapshot.StatusChoices.values)
+
+
+def normalize_snapshot_status(status: str | None) -> str | None:
+    value = str(status or "").strip().lower()
+    if not value:
+        return None
+
+    valid_statuses = snapshot_status_values()
+    if value not in valid_statuses:
+        raise ValueError(f"Invalid snapshot status: {status}. Expected one of: {', '.join(valid_statuses)}")
+    return value
+
+
+def filter_snapshots_by_status(queryset: QuerySet, status: str | None) -> QuerySet:
+    value = normalize_snapshot_status(status)
+    return queryset.filter(status=value) if value else queryset

archivebox/core/sqlite_backend/base.py

@@ -45,26 +45,14 @@ def _format_sql(query: str, params=None) -> str:
 def _log_locked_database(query: str, params=None, *, attempt: int, elapsed: float, retry_interval: float) -> None:
     from rich.console import Console
 
-    from archivebox.misc.db import sqlite_lock_holders
+    from archivebox.misc.db import log_sqlite_lock_holders
 
     console = Console(stderr=True)
     console.print(
         f"[yellow][*] SQLite database is locked for {elapsed:.0f}s; retrying in {retry_interval:g}s... attempt={attempt}[/yellow]",
     )
     console.print(f"[yellow]    Query: {_format_sql(query, params)}[/yellow]")
-    holders = sqlite_lock_holders()
-    if holders:
-        console.print("[yellow]    DB holders:[/yellow]")
-        for holder in holders[:8]:
-            console.print(f"[yellow]    - {holder}[/yellow]")
-        if len(holders) > 8:
-            console.print(f"[yellow]    ... {len(holders) - 8} more[/yellow]")
-    else:
-        console.print("[yellow]    No local process with index.sqlite3 open was visible to this user.[/yellow]")
-    if attempt == 1:
-        console.print(
-            "[dim]    SQLite does not expose the active SQL statement from another process; only local PIDs with the DB open can be shown.[/dim]",
-        )
+    log_sqlite_lock_holders(console)
 
 
 def _connection_in_transaction(connection) -> bool:

archivebox/core/tag_util.py

@@ -12,7 +12,8 @@ from django.http import HttpRequest
 from django.urls import reverse
 
 from archivebox.config.common import get_config
-from archivebox.core.host_util import build_snapshot_url, build_web_url
+from archivebox.misc.util import sanitize_html_text
+from archivebox.core.routes_util import build_snapshot_url, build_web_url
 from archivebox.core.models import Snapshot, SnapshotTag, Tag
 
 
@@ -33,7 +34,7 @@ TAG_HAS_SNAPSHOTS_CHOICES = (
 
 
 def normalize_tag_name(name: str) -> str:
-    return (name or "").strip()
+    return sanitize_html_text(name).strip()
 
 
 def normalize_tag_sort(sort: str = "created_desc") -> str:
@@ -61,7 +62,6 @@ def get_matching_tags(
     created_by: str = "",
     year: str = "",
     has_snapshots: str = "all",
-    with_snapshot_counts: bool = True,
 ) -> QuerySet[Tag]:
     sort = normalize_tag_sort(sort)
     has_snapshots = normalize_has_snapshots_filter(has_snapshots)
@@ -115,9 +115,9 @@ def add_snapshot_counts(tags: list[Tag], snapshot_queryset: QuerySet[Snapshot] |
     queryset = SnapshotTag.objects.filter(tag_id__in=tag_ids)
     if snapshot_queryset is not None:
         queryset = queryset.filter(snapshot_id__in=snapshot_queryset.values("id"))
-    counts = {row["tag_id"]: row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))}
+    counts = {int(row["tag_id"]): row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))}
     for tag in tags:
-        tag.num_snapshots = counts.get(tag.pk, 0)
+        tag.num_snapshots = counts.get(int(tag.pk), 0)
 
 
 def get_tag_creator_choices() -> list[tuple[str, str]]:
@@ -247,7 +247,7 @@ def _build_snapshot_preview_map(
 
 
 def build_tag_card(tag: Tag, snapshot_previews: list[dict[str, Any]] | None = None) -> dict[str, Any]:
-    count = getattr(tag, "num_snapshots", None)
+    count = tag.__dict__.get("num_snapshots")
     if count is None:
         count = tag.snapshot_set.count()
     return {
@@ -284,7 +284,6 @@ def build_tag_cards(
         created_by=created_by,
         year=year,
         has_snapshots=has_snapshots,
-        with_snapshot_counts=needs_snapshot_count_annotation,
     )
     if limit is not None:
         queryset = queryset[:limit]

archivebox/core/takeover_util.py

@@ -18,10 +18,18 @@ def runtime_stack_owner_types():
     from archivebox.machine.models import Process
 
     return (
-        Process.TypeChoices.UPDATE,
         Process.TypeChoices.SERVER,
         Process.TypeChoices.ORCHESTRATOR,
+    )
+
+
+def foreground_runner_owner_types():
+    from archivebox.machine.models import Process
+
+    return (
+        Process.TypeChoices.SERVER,
         Process.TypeChoices.ADD,
+        Process.TypeChoices.UPDATE,
     )
 
 
@@ -70,15 +78,13 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
     if exclude_id is not None:
         base_qs = base_qs.exclude(id=exclude_id)
 
-    top_level_types = (
-        Process.TypeChoices.UPDATE,
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ADD,
-    )
     for qs in (
-        base_qs.filter(process_type__in=top_level_types),
+        # Only server parents own HTTP runtime leadership. Foreground add/update
+        # commands can own runner/sonic components, but server startup must never
+        # wait behind them before binding Daphne.
+        base_qs.filter(process_type=Process.TypeChoices.SERVER),
         # A foreground `archivebox run` process is allowed to own the runtime
-        # stack when no server/update/add parent is alive. A runner launched by
+        # stack when no server/add parent is alive. A runner launched by
         # supervisord is only a child worker; after its parent is killed it must
         # not keep stealing leadership from the next foreground command.
         base_qs.filter(process_type=Process.TypeChoices.ORCHESTRATOR).exclude(parent__process_type=Process.TypeChoices.SUPERVISORD),
@@ -95,6 +101,30 @@ def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:
     return bool(owner and owner.id == command.id)
 
 
+def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
+    from archivebox.machine.models import Machine, Process
+
+    machine = Machine.current()
+    qs = Process.objects.filter(
+        machine=machine,
+        status=Process.StatusChoices.RUNNING,
+        pwd=str(data_dir),
+        process_type__in=foreground_runner_owner_types(),
+    )
+    if exclude_id is not None:
+        qs = qs.exclude(id=exclude_id)
+    for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+        if proc.is_running:
+            return proc
+        proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+    return None
+
+
+def command_owns_foreground_runner(command, *, data_dir: str | Path) -> bool:
+    owner = foreground_runner_owner(data_dir=data_dir)
+    return bool(owner and owner.id == command.id)
+
+
 def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
     try:
         from archivebox.config.common import get_config
@@ -108,9 +138,9 @@ def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
     if not names and owner is not None:
         from archivebox.machine.models import Process
 
-        if owner.process_type in {Process.TypeChoices.SERVER, Process.TypeChoices.ADD}:
+        if owner.process_type == Process.TypeChoices.SERVER:
             names = ["orchestrator", "server"]
-        elif owner.process_type in {Process.TypeChoices.UPDATE, Process.TypeChoices.ORCHESTRATOR}:
+        elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
             names = ["orchestrator"]
 
     return ", ".join(dict.fromkeys(names)) or "runtime stack"
@@ -292,3 +322,26 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva
     command.modified_at = timezone.now()
     command.save(update_fields=["modified_at"])
     return {"resumed": announced, "previous_owner_pid": previous_owner_pid}
+
+
+def standby_until_foreground_runner_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> dict[str, object]:
+    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
+
+    announced = False
+    previous_owner_pid = None
+    while not command_owns_foreground_runner(command, data_dir=data_dir):
+        reap_foreground_supervisord_process()
+        if not announced:
+            owner = foreground_runner_owner(data_dir=data_dir)
+            owner_pid = owner.pid if owner else "unknown"
+            previous_owner_pid = owner_pid
+            rprint(
+                f"[yellow][*] A newer archivebox process took over the orchestrator, sonic "
+                f"(pid={owner_pid}). Work will continue there, and will resume here if that process exits and work still remains.[/yellow]",
+                file=sys.stderr,
+            )
+            announced = True
+        time.sleep(interval)
+    command.modified_at = timezone.now()
+    command.save(update_fields=["modified_at"])
+    return {"resumed": announced, "previous_owner_pid": previous_owner_pid}

archivebox/core/templatetags/core_tags.py

@@ -4,18 +4,22 @@ from django import template
 from django.contrib.admin.templatetags.base import InclusionAdminNode
 from django.utils.safestring import mark_safe
 from django.utils.html import escape
+from django.templatetags.static import static
+from django.utils import timezone
+from django.utils.text import Truncator
 
 from pathlib import Path
 
-from archivebox.hooks import (
+from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target
+
+from archivebox.plugins.discovery import (
     get_plugin_icon,
     get_plugin_template,
     get_plugin_name,
 )
-from archivebox.core.host_util import (
+from archivebox.core.routes_util import (
     canonical_base_host_for_request,
     get_admin_base_url,
-    get_public_base_url,
     get_web_base_url,
     get_snapshot_base_url,
     build_snapshot_url,
@@ -27,7 +31,6 @@ register = template.Library()
 _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml", ".yaml", ".md", ".log")
 _IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
 _MHTML_PREVIEW_EXTS = (".mhtml", ".mht")
-_WACZ_PREVIEW_EXTS = (".wacz", ".warc", ".warc.gz")
 
 _MEDIA_FILE_EXTS = {
     ".mp4",
@@ -75,6 +78,12 @@ def _normalize_output_files(output_files: Any) -> dict[str, dict[str, Any]]:
     return {}
 
 
+def _snapshot_id(value: Any) -> Any:
+    from archivebox.core.models import Snapshot
+
+    return value.id if isinstance(value, Snapshot) else value
+
+
 def _coerce_output_file_size(value: Any) -> int | None:
     try:
         return max(int(value or 0), 0)
@@ -84,7 +93,7 @@ def _coerce_output_file_size(value: Any) -> int | None:
 
 def _count_media_files(result) -> int:
     try:
-        output_files = _normalize_output_files(getattr(result, "output_files", None) or {})
+        output_files = _normalize_output_files(result.output_files or {})
     except Exception:
         output_files = {}
 
@@ -120,7 +129,7 @@ def _list_media_files(result) -> list[dict]:
     except Exception:
         return media_files
 
-    output_files = _normalize_output_files(getattr(result, "output_files", None) or {})
+    output_files = _normalize_output_files(result.output_files or {})
     candidates: list[tuple[Path, int | None]] = []
     if output_files:
         for path, metadata in output_files.items():
@@ -210,7 +219,7 @@ def _build_snapshot_preview_url(snapshot_id: str, path: str = "", request=None,
         _is_text_preview_path(path)
         or _is_image_preview_path(path)
         or (path or "").lower().endswith(_MHTML_PREVIEW_EXTS)
-        or (path or "").lower().endswith(_WACZ_PREVIEW_EXTS)
+        or is_archivewebpage_replay_target(path or "")
     ):
         return url
     separator = "&" if "?" in url else "?"
@@ -299,7 +308,7 @@ def result_list(context, cl):
     """
     num_sorted_fields = 0
     request = context.get("request")
-    config = getattr(request, "archivebox_config", None) or context.get("CONFIG")
+    config = request.__dict__.get("archivebox_config") if request is not None else context.get("CONFIG")
     results = cl.result_list
     if config is not None:
         for obj in results:
@@ -442,7 +451,7 @@ def _unconfigured_banner_context(request) -> dict:
     # operator's clipboard already aligned with subdomain routing. The config
     # parser strips the leading ``*.`` so users can paste it verbatim.
     suggested_base_url = f"{scheme}://*.{canonical_host}" if canonical_host else ""
-    user = getattr(request, "user", None)
+    user = request.user
     is_superuser = bool(user and user.is_authenticated and user.is_superuser)
     machine_admin_url = ""
     if is_superuser:
@@ -450,7 +459,7 @@ def _unconfigured_banner_context(request) -> dict:
             from archivebox.machine.models import Machine
 
             machine = Machine.current()
-            machine_admin_url = f"/admin/machine/machine/{machine.id.hex}/change/"
+            machine_admin_url = f"/admin/machine/machine/{machine.id}/change/"
         except Exception:
             machine_admin_url = ""
     return {
@@ -488,26 +497,217 @@ def web_base_url(context) -> str:
     return get_web_base_url(request=context.get("request"), config=context.get("CONFIG"))
 
 
-@register.simple_tag(takes_context=True)
-def public_base_url(context) -> str:
-    return get_public_base_url(request=context.get("request"), config=context.get("CONFIG"))
-
-
 @register.simple_tag(takes_context=True)
 def snapshot_base_url(context, snapshot) -> str:
-    snapshot_id = getattr(snapshot, "id", snapshot)
+    snapshot_id = _snapshot_id(snapshot)
     return get_snapshot_base_url(str(snapshot_id), request=context.get("request"), config=context.get("CONFIG"))
 
 
 @register.simple_tag(takes_context=True)
 def snapshot_url(context, snapshot, path: str = "") -> str:
-    snapshot_id = getattr(snapshot, "id", snapshot)
+    snapshot_id = _snapshot_id(snapshot)
     return build_snapshot_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
 
 
+@register.simple_tag(takes_context=True)
+def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> str:
+    snapshot_id = str(_snapshot_id(snapshot))
+    url_cache = snapshot.__dict__.setdefault("_snapshot_archiveresult_url_cache", {})
+    cache_key = (plugin, filename)
+    if cache_key in url_cache:
+        return url_cache[cache_key]
+
+    results = None
+    if "_admin_archiveresults" in snapshot.__dict__:
+        results = snapshot.__dict__["_admin_archiveresults"]
+    elif "archiveresult_set" in snapshot.__dict__.get("_prefetched_objects_cache", {}):
+        results = snapshot.archiveresult_set.all()
+    elif "_snapshot_archiveresult_url_results" in snapshot.__dict__:
+        results = snapshot.__dict__["_snapshot_archiveresult_url_results"]
+
+    if results is None:
+        from archivebox.core.models import ArchiveResult
+
+        results = list(
+            ArchiveResult.objects.filter(
+                snapshot_id=snapshot_id,
+                plugin__in=("screenshot", "chrome_extension_screenshot", "favicon"),
+                status=ArchiveResult.StatusChoices.SUCCEEDED,
+            ).only(
+                "plugin",
+                "status",
+                "output_files",
+            ),
+        )
+        snapshot.__dict__["_snapshot_archiveresult_url_results"] = results
+
+    for result in results:
+        if result.plugin != plugin or result.status != "succeeded":
+            continue
+        output_files = result.output_files or {}
+        file_info = output_files.get(filename)
+        if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
+            continue
+        output_path = filename if file_info.get("root_relative") else f"{plugin}/{filename}"
+        url_cache[cache_key] = build_snapshot_url(snapshot_id, output_path, request=context.get("request"), config=context.get("CONFIG"))
+        return url_cache[cache_key]
+
+    url_cache[cache_key] = ""
+    return ""
+
+
+@register.simple_tag(takes_context=True)
+def snapshot_index_row(context, link) -> str:
+    snapshot_id = str(_snapshot_id(link))
+    request = context.get("request")
+    config = context.get("CONFIG")
+    snapshot_base = get_snapshot_base_url(snapshot_id, request=request, config=config)
+
+    status = getattr(link, "status", None) or "unknown"
+    bookmarked_at = getattr(link, "bookmarked_at", None)
+    timestamp = getattr(link, "timestamp", "")
+    if bookmarked_at:
+        bookmarked_at = timezone.localtime(bookmarked_at)
+        date_text = bookmarked_at.strftime("%Y-%m-%d")
+        time_text = bookmarked_at.strftime("%H:%M")
+        sort_value = str(bookmarked_at.timestamp())
+        title_time = f"Bookmarked: {bookmarked_at} ({timestamp})"
+    else:
+        date_text = ""
+        time_text = ""
+        sort_value = ""
+        title_time = f"Bookmarked: ({timestamp})"
+
+    url = getattr(link, "url", "") or ""
+    title = getattr(link, "title", "") or ""
+    is_pending = status in {"queued", "started", "backoff"}
+    title_text = title or ("Loading..." if is_pending else url)
+    tags_str = link.tags_str() if callable(getattr(link, "tags_str", None)) else getattr(link, "tags_str", "")
+    tag_html = "".join(f'<span class="snapshot-tag">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag)
+    if tag_html:
+        tag_cell = f'<span class="snapshot-tags">{tag_html}</span>'
+    else:
+        tag_cell = '<span class="empty-value">...</span>'
+
+    num_outputs = int(getattr(link, "num_outputs", 0) or 0)
+    icons = link.icons() if callable(getattr(link, "icons", None)) else getattr(link, "icons", "")
+    icons_cell = str(icons) if icons else '<span class="empty-value">...</span>'
+    archive_size = int(getattr(link, "archive_size", 0) or 0)
+    size_cell = file_size(archive_size) if archive_size else '<span class="empty-value">...</span>'
+    output_plural = "" if num_outputs == 1 else "s"
+
+    if is_pending:
+        preview_html = (
+            '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress">'
+            f'<img src="{escape(static("spinner.gif"))}" alt="" decoding="async" loading="lazy">'
+            "</span>"
+        )
+    elif "_public_preview_paths" in link.__dict__:
+        preview_paths = list(getattr(link, "_public_preview_paths", []) or [])
+        if preview_paths:
+            preview_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in preview_paths]
+            preview_html = (
+                f'<img src="{escape(preview_urls[0])}" '
+                f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
+                'onerror="nextPublicSnapshotPreview(this)" class="snapshot-preview screenshot" alt="" decoding="async" loading="lazy">'
+            )
+        else:
+            preview_html = '<span class="snapshot-preview snapshot-preview-empty" aria-label="No preview available"></span>'
+    else:
+        preview_urls = [
+            url
+            for url in (
+                snapshot_archiveresult_url(context, link, "screenshot", "screenshot.png"),
+                snapshot_archiveresult_url(context, link, "chrome_extension_screenshot", "screenshot-1.png"),
+                snapshot_archiveresult_url(context, link, "chrome_extension_screenshot", "screenshot.png"),
+            )
+            if url
+        ]
+        if preview_urls:
+            preview_html = (
+                f'<img src="{escape(preview_urls[0])}" '
+                f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
+                'onerror="nextPublicSnapshotPreview(this)" class="snapshot-preview screenshot" alt="" decoding="async" loading="lazy">'
+            )
+        else:
+            preview_html = '<span class="snapshot-preview snapshot-preview-empty" aria-label="No preview available"></span>'
+
+    if "_public_favicon_paths" in link.__dict__:
+        favicon_paths = list(getattr(link, "_public_favicon_paths", []) or [])
+        if favicon_paths:
+            favicon_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in favicon_paths]
+            favicon_html = (
+                f'<img src="{escape(favicon_urls[0])}" '
+                f'data-fallbacks="{escape(",".join(favicon_urls[1:]))}" '
+                'onerror="nextPublicSnapshotPreview(this)" '
+                'class="link-favicon" alt="" decoding="async" loading="lazy">'
+            )
+        else:
+            favicon_html = '<span class="link-favicon link-favicon-empty" aria-hidden="true"></span>'
+    else:
+        favicon_url = snapshot_archiveresult_url(context, link, "favicon", "favicon.ico")
+        favicon_html = (
+            (
+                f'<img src="{escape(favicon_url)}" '
+                'data-fallbacks="" '
+                'onerror="nextPublicSnapshotPreview(this)" '
+                'class="link-favicon" alt="" decoding="async" loading="lazy">'
+            )
+            if favicon_url
+            else '<span class="link-favicon link-favicon-empty" aria-hidden="true"></span>'
+        )
+
+    html = f"""
+<tr class="snapshot-row status-{escape(status)}">
+    <td class="snapshot-time" title="{escape(title_time)}" data-sort="{escape(sort_value)}">
+        <a href="{escape(snapshot_base)}/index.html">
+            <span>{escape(date_text)}</span>
+            <small>{escape(time_text)}</small>
+        </a>
+    </td>
+    <td class="snapshot-preview-cell">
+        <a href="{escape(snapshot_base)}/index.html" title="Open archived snapshot">
+            {preview_html}
+        </a>
+    </td>
+    <td class="snapshot-title-cell" title="{escape(title or url)}">
+        <div class="snapshot-title-line">
+            <a href="{escape(snapshot_base)}/index.html" class="snapshot-favicon-link" title="Open archived snapshot">
+                {favicon_html}
+            </a>
+            <a href="{escape(snapshot_base)}/index.html" class="snapshot-title">
+                {escape(Truncator(title_text).chars(110))}
+            </a>
+        </div>
+        <a href="{escape(url)}" class="snapshot-url" title="{escape(url)}" target="_blank" rel="noopener noreferrer">
+            {escape(url)}
+        </a>
+    </td>
+    <td class="snapshot-tags-cell">
+        {tag_cell}
+    </td>
+    <td class="snapshot-status-cell">
+        <span class="snapshot-status">{escape(status)}</span>
+    </td>
+    <td class="snapshot-files-cell">
+        <span data-number-for="{escape(url)}" title="{num_outputs} successful outputs">
+            {icons_cell}
+        </span>
+    </td>
+    <td class="snapshot-size-cell">
+        <a href="{escape(snapshot_base)}/?files=1" title="View archived files">
+            {size_cell}
+        </a>
+        <small>{num_outputs} output{output_plural}</small>
+    </td>
+</tr>
+"""
+    return mark_safe(html)
+
+
 @register.simple_tag(takes_context=True)
 def snapshot_preview_url(context, snapshot, path: str = "") -> str:
-    snapshot_id = getattr(snapshot, "id", snapshot)
+    snapshot_id = _snapshot_id(snapshot)
     return _build_snapshot_preview_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
 
 
@@ -537,16 +737,18 @@ def plugin_card(context, result) -> str:
         - output_path: Path to output relative to snapshot dir (from embed_path())
         - plugin: Plugin base name
     """
-    if result is None or not hasattr(result, "plugin"):
+    from archivebox.core.models import ArchiveResult
+
+    if result is None or not isinstance(result, ArchiveResult):
         return ""
 
     plugin = get_plugin_name(result.plugin)
     template_str = get_plugin_template(plugin, "card")
 
     # Use embed_path() for the display path
-    raw_output_path = result.embed_path() if hasattr(result, "embed_path") else ""
+    raw_output_path = result.embed_path() or ""
     output_url = build_snapshot_url(
-        str(getattr(result, "snapshot_id", "")),
+        str(result.snapshot_id),
         raw_output_path or "",
         request=context.get("request"),
         config=context.get("CONFIG"),
@@ -557,7 +759,7 @@ def plugin_card(context, result) -> str:
     media_file_count = _count_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else 0
     media_files = _list_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
     if media_files:
-        snapshot_id = str(getattr(result, "snapshot_id", ""))
+        snapshot_id = str(result.snapshot_id)
         request = context.get("request")
         config = context.get("CONFIG")
         for item in media_files:
@@ -591,7 +793,7 @@ def plugin_card(context, result) -> str:
         pass
 
     if force_text_preview:
-        preview = _render_text_file_preview(getattr(result, "snapshot_dir", None), raw_output_path, plugin, icon_html)
+        preview = _render_text_file_preview(result.snapshot_dir, raw_output_path, plugin, icon_html)
         if preview:
             return mark_safe(preview)
 
@@ -607,7 +809,7 @@ def plugin_card(context, result) -> str:
 def output_card(snapshot, output_path: str, plugin: str) -> str:
     plugin_name = get_plugin_name(plugin)
     icon_html = get_plugin_icon(plugin_name)
-    preview = _render_text_file_preview(getattr(snapshot, "output_dir", None), output_path, plugin_name, icon_html)
+    preview = _render_text_file_preview(snapshot.output_dir, output_path, plugin_name, icon_html)
     if preview:
         return mark_safe(preview)
 
@@ -623,7 +825,9 @@ def plugin_full(context, result) -> str:
 
     Usage: {% plugin_full result %}
     """
-    if result is None or not hasattr(result, "plugin"):
+    from archivebox.core.models import ArchiveResult
+
+    if result is None or not isinstance(result, ArchiveResult):
         return ""
 
     plugin = get_plugin_name(result.plugin)
@@ -633,14 +837,13 @@ def plugin_full(context, result) -> str:
         return ""
 
     raw_output_path = ""
-    if hasattr(result, "embed_path_db"):
-        raw_output_path = result.embed_path_db() or ""
-    if not raw_output_path and hasattr(result, "embed_path"):
+    raw_output_path = result.embed_path_db() or ""
+    if not raw_output_path:
         raw_output_path = result.embed_path() or ""
     if _is_root_snapshot_output_path(raw_output_path):
         return ""
     output_url = build_snapshot_url(
-        str(getattr(result, "snapshot_id", "")),
+        str(result.snapshot_id),
         raw_output_path,
         request=context.get("request"),
         config=context.get("CONFIG"),
@@ -684,7 +887,7 @@ def api_token(context) -> str:
     from archivebox.api.auth import get_or_create_api_token
 
     request = context.get("request")
-    user = getattr(request, "user", None)
+    user = request.user
     if not user or not user.is_authenticated:
         return ""
 

archivebox/core/urls.py

@@ -19,13 +19,16 @@ from archivebox.core.views import (
     SnapshotView,
     SnapshotPathView,
     SnapshotReplayView,
+    SnapshotReplayAuthView,
     OriginalDomainReplayView,
     PublicIndexView,
     AddView,
     WebAddView,
     HealthCheckView,
-    live_progress_view,
 )
+from archivebox.progressmonitor.views import live_progress_view
+from archivebox.search.views import public_snapshot_search_stream_view
+from abx_plugins.plugins.opencode.views import opencode_proxy_view
 
 
 # GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
@@ -41,6 +44,9 @@ urlpatterns = [
     path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}),
     path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}),
     path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"),
+    re_path(r"^admin/agent/?(?=$|opencode)", include("abx_plugins.plugins.opencode.urls")),
+    re_path(r"^(?P<path>assets/.*)$", opencode_proxy_view, name="opencode-assets"),
+    path("public/search-stream/", public_snapshot_search_stream_view, name="public-search-stream"),
     path("public/", PublicIndexView.as_view(), name="public-index"),
     path("public.html", RedirectView.as_view(url="/public/"), name="public-index-html"),
     path("archive/", RedirectView.as_view(url="/")),
@@ -65,6 +71,7 @@ urlpatterns = [
         name="snapshot-path-nodate",
     ),
     path("admin/core/snapshot/add/", RedirectView.as_view(url="/add/")),
+    path("admin/core/snapshot/replay-auth/", SnapshotReplayAuthView.as_view(), name="snapshot-replay-auth"),
     path("add/", AddView.as_view(), name="add"),
     # ``query_string=True`` preserves the ``?next=…`` param that Django's
     # auth/login mixins append, so e.g. ``UserPassesTestMixin`` redirecting

archivebox/core/widgets.py

@@ -4,6 +4,8 @@ import json
 import re
 import hashlib
 from django import forms
+from django.db.models.manager import BaseManager
+from django.db.models.query import QuerySet
 from django.utils.html import escape
 from django.utils.safestring import mark_safe
 
@@ -38,6 +40,16 @@ class TagEditorWidget(forms.Widget):
             normalized = f"t_{normalized}"
         return normalized
 
+    def _json_for_inline_script(self, value):
+        """Serialize JSON so it cannot close the surrounding inline <script> tag."""
+        return json.dumps(value).translate(
+            {
+                ord(">"): "\\u003E",
+                ord("<"): "\\u003C",
+                ord("&"): "\\u0026",
+            },
+        )
+
     def _tag_style(self, value):
         """Compute a stable pastel color style for a tag value."""
         tag = (value or "").strip().lower()
@@ -65,10 +77,12 @@ class TagEditorWidget(forms.Widget):
         # Parse value to get list of tag names
         tags = []
         if value:
-            if hasattr(value, "all"):  # QuerySet
+            if isinstance(value, (BaseManager, QuerySet)):
                 tags = sorted([tag.name for tag in value.all()])
             elif isinstance(value, (list, tuple)):
-                if value and hasattr(value[0], "name"):  # List of Tag objects
+                from archivebox.core.models import Tag
+
+                if value and isinstance(value[0], Tag):  # List of Tag objects
                     tags = sorted([tag.name for tag in value])
                 else:  # List of strings or IDs
                     # Could be tag IDs from form submission
@@ -102,6 +116,8 @@ class TagEditorWidget(forms.Widget):
                 </span>
             '''
 
+        tags_json = self._json_for_inline_script(tags)
+
         # Build the widget HTML
         html = f'''
         <div id="{widget_id}_container" class="tag-editor-container" onclick="focusTagInput_{widget_id}(event)">
@@ -124,7 +140,7 @@ class TagEditorWidget(forms.Widget):
 
         <script>
         (function() {{
-            var currentTags_{widget_id} = {json.dumps(tags)};
+            var currentTags_{widget_id} = {tags_json};
             var autocompleteTimeout_{widget_id} = null;
 
             window.focusTagInput_{widget_id} = function(event) {{
@@ -675,12 +691,14 @@ class InlineTagEditorWidget(TagEditorWidget):
         # Parse value to get list of tag dicts with id and name
         tag_data = []
         if value:
-            if hasattr(value, "all"):  # QuerySet
+            if isinstance(value, (BaseManager, QuerySet)):
                 for tag in value.all():
                     tag_data.append({"id": tag.pk, "name": tag.name})
                 tag_data.sort(key=lambda x: x["name"].lower())
             elif isinstance(value, (list, tuple)):
-                if value and hasattr(value[0], "name"):
+                from archivebox.core.models import Tag
+
+                if value and isinstance(value[0], Tag):
                     for tag in value:
                         tag_data.append({"id": tag.pk, "name": tag.name})
                     tag_data.sort(key=lambda x: x["name"].lower())

archivebox/crawls/admin.py

@@ -1,27 +1,28 @@
 __package__ = "archivebox.crawls"
 
 from copy import copy
-from urllib.parse import urlencode
+import json
+from urllib.parse import urlencode, urlparse
 
 from django import forms
 from django.core.paginator import Paginator
 from django.http import JsonResponse, HttpRequest, HttpResponseBadRequest, HttpResponseNotAllowed
 from django.shortcuts import get_object_or_404, redirect
+from django.template.response import TemplateResponse
 from django.template.loader import render_to_string
 from django.urls import path, reverse
 from django.utils.html import escape, format_html, format_html_join
 from django.utils import timezone
 from django.utils.safestring import mark_safe
 from django.contrib import admin, messages
-from django.db.models import Case, CharField, Count, IntegerField, OuterRef, Q, Subquery, Value, When
-from django.db.models.functions import Coalesce
+from django.db.models import Count, F, Q
 
 
 from django_object_actions import action
 
 from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 
-from archivebox.core.models import Snapshot
+from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
     PERMISSIONS_META,
@@ -34,6 +35,7 @@ from archivebox.core.permissions import (
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
 from archivebox.crawls.models import Crawl, CrawlSchedule
 from archivebox.misc.paginators import AcceleratedPaginator
+from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.workers.models import RETRY_AT_MAX
 
 
@@ -70,22 +72,15 @@ def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50,
     if status_filter in valid_statuses:
         filtered_qs = filtered_qs.filter(status=status_filter)
 
+    # Keep ArchiveResult counters as scalar subqueries so the paginated
+    # Snapshot queryset does not become a join+GROUP BY over every result row.
     snapshots_qs = filtered_qs.order_by("-created_at").annotate(
-        total_results=Count("archiveresult"),
-        succeeded_results=Count("archiveresult", filter=Q(archiveresult__status="succeeded")),
-        failed_results=Count("archiveresult", filter=Q(archiveresult__status="failed")),
-        started_results=Count("archiveresult", filter=Q(archiveresult__status="started")),
-        skipped_results=Count("archiveresult", filter=Q(archiveresult__status="skipped")),
-        snapshot_permissions=Case(
-            When(permissions=PERMISSIONS_PUBLIC, then=Value(PERMISSIONS_PUBLIC)),
-            When(permissions=PERMISSIONS_UNLISTED, then=Value(PERMISSIONS_UNLISTED)),
-            When(permissions=PERMISSIONS_PRIVATE, then=Value(PERMISSIONS_PRIVATE)),
-            When(crawl__permissions=PERMISSIONS_PUBLIC, then=Value(PERMISSIONS_PUBLIC)),
-            When(crawl__permissions=PERMISSIONS_UNLISTED, then=Value(PERMISSIONS_UNLISTED)),
-            When(crawl__permissions=PERMISSIONS_PRIVATE, then=Value(PERMISSIONS_PRIVATE)),
-            default=Value(PERMISSIONS_PRIVATE),
-            output_field=CharField(),
-        ),
+        total_results=ArchiveResult.snapshot_count_expr(),
+        succeeded_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
+        failed_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.FAILED),
+        started_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.STARTED),
+        skipped_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SKIPPED),
+        snapshot_permissions=F("permissions"),
     )
 
     page_number = request.GET.get(page_param, 1) if request is not None else 1
@@ -260,7 +255,7 @@ def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50,
                                         background: {progress_color};
                                         transition: width 0.3s;"></div>
                         </div>
-                        <a href="/admin/core/archiveresult/?snapshot__id__exact={snapshot.id.hex}"
+                        <a href="/admin/core/archiveresult/?snapshot__id__exact={snapshot.id}"
                            style="font-size: 11px; color: #417690; min-width: 35px; text-decoration: none;"
                            title="View archive results">{progress_text}</a>
                     </div>
@@ -381,7 +376,7 @@ class URLFiltersField(forms.Field):
     def to_python(self, value):
         if isinstance(value, dict):
             return value
-        return {"allowlist": "", "denylist": "", "same_domain_only": False, "subpaths_only": False}
+        return {"allowlist": "", "denylist": "", "same_domain_only": False, "subpaths_only": False, "only_new": False}
 
 
 class CrawlAdminForm(forms.ModelForm):
@@ -423,13 +418,140 @@ class CrawlAdminForm(forms.ModelForm):
         config = dict(self.instance.config or {}) if self.instance and self.instance.pk else {}
         if self.instance and self.instance.pk:
             self.initial["tags_editor"] = self.instance.tags_str
+        effective_only_new = self.effective_only_new(self.instance if self.instance and self.instance.pk else None)
+        derived_filter_toggles = self.derive_filter_toggles(
+            self.instance.urls if self.instance and self.instance.pk else "",
+            config.get("URL_ALLOWLIST", ""),
+        )
         self.initial["url_filters"] = {
             "allowlist": config.get("URL_ALLOWLIST", ""),
             "denylist": config.get("URL_DENYLIST", ""),
-            "same_domain_only": False,
-            "subpaths_only": False,
+            "same_domain_only": derived_filter_toggles["same_domain_only"],
+            "subpaths_only": derived_filter_toggles["subpaths_only"],
+            "only_new": effective_only_new,
         }
 
+    @staticmethod
+    def extract_url_line(line):
+        line = str(line or "").strip()
+        if not line or line.startswith("#"):
+            return ""
+        if line.startswith("{"):
+            try:
+                return str(json.loads(line).get("url", "")).strip()
+            except (TypeError, ValueError, json.JSONDecodeError):
+                return ""
+        return line
+
+    @staticmethod
+    def regex_escape(text):
+        escaped = ""
+        for char in str(text or ""):
+            escaped += f"\\{char}" if char in r".*+?^${}()|[]\\" else char
+        return escaped
+
+    @classmethod
+    def generated_host_allowlist(cls, urls):
+        seen = set()
+        domains = []
+        for raw_line in str(urls or "").splitlines():
+            url = cls.extract_url_line(raw_line)
+            if not url:
+                continue
+            parsed = urlparse(url)
+            domain = (parsed.hostname or "").lower()
+            if not domain or domain in seen:
+                continue
+            seen.add(domain)
+            domains.append(domain)
+        if not domains:
+            return ""
+        return "^https?://(" + "|".join(cls.regex_escape(domain) for domain in domains) + ")([:/]|$)"
+
+    @staticmethod
+    def subpath_prefix(pathname):
+        path = str(pathname or "/")
+        while "//" in path:
+            path = path.replace("//", "/")
+        if not path or path == "/":
+            return "/"
+        if path.endswith("/"):
+            return path
+        last_slash = path.rfind("/")
+        last_part = path[last_slash + 1 :]
+        if "." in last_part:
+            return path[: last_slash + 1] or "/"
+        return path
+
+    @staticmethod
+    def parsed_host_and_port(parsed):
+        host = (parsed.hostname or "").lower()
+        if not host:
+            return ""
+        try:
+            port = parsed.port
+        except ValueError:
+            port = None
+        return f"{host}:{port}" if port is not None else host
+
+    @classmethod
+    def generated_subpath_allowlist(cls, urls):
+        seen = set()
+        paths = []
+        for raw_line in str(urls or "").splitlines():
+            url = cls.extract_url_line(raw_line)
+            if not url:
+                continue
+            parsed = urlparse(url)
+            domain = (parsed.hostname or "").lower()
+            if domain:
+                seen.add(domain)
+            host = cls.parsed_host_and_port(parsed)
+            path = cls.subpath_prefix(parsed.path)
+            path_key = f"{host}{path}"
+            if not host or path_key in seen:
+                continue
+            seen.add(path_key)
+            paths.append((host, path))
+        if not paths:
+            return ""
+        patterns = []
+        for host, path in paths:
+            if path == "/":
+                patterns.append(f"^https?://{cls.regex_escape(host)}([/?#]|$)")
+            elif path.endswith("/"):
+                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}")
+            else:
+                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}([/?#]|$)")
+        return "\n".join(patterns)
+
+    @classmethod
+    def derive_filter_toggles(cls, urls, allowlist):
+        normalized_allowlist = "\n".join(Crawl.split_filter_patterns(allowlist))
+        if not normalized_allowlist:
+            return {"same_domain_only": False, "subpaths_only": False}
+        if normalized_allowlist == cls.generated_subpath_allowlist(urls):
+            return {"same_domain_only": True, "subpaths_only": True}
+        if normalized_allowlist == cls.generated_host_allowlist(urls):
+            return {"same_domain_only": True, "subpaths_only": False}
+        return {"same_domain_only": False, "subpaths_only": False}
+
+    @staticmethod
+    def effective_only_new(crawl=None):
+        from archivebox.config.common import get_config
+
+        if crawl is not None:
+            return bool(get_config(crawl=crawl, resolve_plugins=False).ONLY_NEW)
+        return bool(get_config(resolve_plugins=False).ONLY_NEW)
+
+    @staticmethod
+    def inherited_only_new(crawl):
+        crawl_without_only_new = copy(crawl)
+        config = dict(crawl.config or {})
+        config.pop("ONLY_NEW", None)
+        crawl_without_only_new.config = config
+        return CrawlAdminForm.effective_only_new(crawl_without_only_new)
+
     def clean_tags_editor(self):
         tags_str = self.cleaned_data.get("tags_editor", "")
         tag_names = []
@@ -452,6 +574,7 @@ class CrawlAdminForm(forms.ModelForm):
             "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
             "same_domain_only": bool(value.get("same_domain_only")),
             "subpaths_only": bool(value.get("subpaths_only")),
+            "only_new": bool(value.get("only_new")),
         }
 
     def save(self, commit=True):
@@ -463,12 +586,18 @@ class CrawlAdminForm(forms.ModelForm):
                 url_filters.get("allowlist", ""),
                 url_filters.get("denylist", ""),
             )
+            config = dict(instance.config or {})
+            only_new = bool(url_filters.get("only_new"))
+            inherited_only_new = self.inherited_only_new(instance)
+            if only_new != inherited_only_new:
+                config["ONLY_NEW"] = only_new
+            else:
+                config.pop("ONLY_NEW", None)
+            instance.config = config
         if commit:
             instance.save()
             instance.apply_crawl_config_filters()
-            save_m2m = getattr(self, "_save_m2m", None)
-            if callable(save_m2m):
-                save_m2m()
+            self._save_m2m()
         return instance
 
 
@@ -590,7 +719,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         super().__init__(model, admin_site)
         self.crawl_admin_base_config = None
         self.stop_reason_cache = {}
-        self.persona_limit_config_cache = {}
 
     class Media:
         css = {"all": ("admin/crawls/crawl_change.css",)}
@@ -600,9 +728,10 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         self.request = request
         self.crawl_admin_base_config = request.archivebox_config
         self.stop_reason_cache = {}
-        self.persona_limit_config_cache = {}
         response = super().changelist_view(request, extra_context)
-        cl = getattr(response, "context_data", {}).get("cl") if hasattr(response, "context_data") else None
+        if not isinstance(response, TemplateResponse):
+            return response
+        cl = response.context_data.get("cl")
         if cl is not None and not self.should_annotate_snapshot_counts(request):
             self.hydrate_visible_snapshot_counts(cl.result_list)
         return response
@@ -623,19 +752,11 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         crawl_ids = [crawl.pk for crawl in crawl_list]
         if not crawl_ids:
             return
-        counts = {
-            str(row["crawl_id"]): row
-            for row in Snapshot.objects.filter(crawl_id__in=crawl_ids)
-            .values("crawl_id")
-            .annotate(
-                num_snapshots_cached=Count("pk"),
-                num_archived_snapshots_cached=Count("pk", filter=Q(status=Snapshot.StatusChoices.SEALED)),
-            )
-        }
+        counts = Snapshot.crawl_total_and_status_counts(crawl_ids, status=Snapshot.StatusChoices.SEALED)
         for crawl in crawl_list:
             row = counts.get(str(crawl.pk), {})
-            crawl.num_snapshots_cached = row.get("num_snapshots_cached", 0)
-            crawl.num_archived_snapshots_cached = row.get("num_archived_snapshots_cached", 0)
+            crawl.num_snapshots_cached = row.get("total", 0)
+            crawl.num_archived_snapshots_cached = row.get("status", 0)
 
     def get_queryset(self, request):
         """Keep joins page-local while computing per-row snapshot counts in the page query."""
@@ -649,25 +770,9 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
             )
         )
         if self.should_annotate_snapshot_counts(request):
-            snapshot_count = (
-                Snapshot.objects.filter(crawl_id=OuterRef("pk")).order_by().values("crawl_id").annotate(count=Count("pk")).values("count")
-            )
-            archived_snapshot_count = (
-                Snapshot.objects.filter(crawl_id=OuterRef("pk"), status=Snapshot.StatusChoices.SEALED)
-                .order_by()
-                .values("crawl_id")
-                .annotate(count=Count("pk"))
-                .values("count")
-            )
             queryset = queryset.annotate(
-                num_snapshots_cached=Coalesce(
-                    Subquery(snapshot_count, output_field=IntegerField()),
-                    Value(0),
-                ),
-                num_archived_snapshots_cached=Coalesce(
-                    Subquery(archived_snapshot_count, output_field=IntegerField()),
-                    Value(0),
-                ),
+                num_snapshots_cached=Snapshot.crawl_count_expr(),
+                num_archived_snapshots_cached=Snapshot.crawl_count_expr(status=Snapshot.StatusChoices.SEALED),
             )
         return queryset
 
@@ -675,7 +780,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         self.request = request
         self.crawl_admin_base_config = request.archivebox_config
         self.stop_reason_cache = {}
-        self.persona_limit_config_cache = {}
         crawl = self.get_object(request, object_id)
         if crawl:
             self.hydrate_visible_snapshot_counts([crawl])
@@ -684,6 +788,13 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
             "crawl_stop_reason": self.stop_reason_for_crawl(crawl) if crawl else "",
             "crawl_snapshots_changelist": self.snapshots_changelist(crawl) if crawl else "",
         }
+        if crawl and crawl.status in {
+            Crawl.StatusChoices.QUEUED,
+            Crawl.StatusChoices.STARTED,
+            Crawl.StatusChoices.PAUSED,
+        }:
+            extra_context["progress_auto_expand"] = True
+            extra_context["progress_endpoint"] = progress_endpoint("crawl", crawl.id)
         return super().change_view(request, object_id, form_url, extra_context)
 
     def add_view(self, request, form_url="", extra_context=None):
@@ -742,7 +853,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         # hold SQLite behind the request for minutes on large archives. The
         # Crawl row is the scheduler signal; the runner observes PAUSED and
         # owns child-row lifecycle work.
-        paused = queryset.exclude(status__in=[Crawl.StatusChoices.SEALED, Crawl.StatusChoices.PAUSED]).update(
+        paused = queryset.exclude(status__in=Crawl.INACTIVE_STATES).update(
             status=Crawl.StatusChoices.PAUSED,
             retry_at=RETRY_AT_MAX,
             modified_at=timezone.now(),
@@ -757,7 +868,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         # Keep resume symmetrical with pause: one tight scheduler UPDATE, no
         # save() hooks and no child fanout in the request path. Paused child
         # rows become runnable through their own resume/maintenance paths.
-        resumed = queryset.filter(status__in=[Crawl.StatusChoices.PAUSED, Crawl.StatusChoices.SEALED]).update(
+        resumed = queryset.filter(status__in=Crawl.INACTIVE_STATES).update(
             status=Crawl.StatusChoices.QUEUED,
             retry_at=timezone.now(),
             modified_at=timezone.now(),
@@ -777,11 +888,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
 
         Snapshot.objects.filter(
             crawl_id__in=crawl_ids,
-            status__in=[
-                Snapshot.StatusChoices.QUEUED,
-                Snapshot.StatusChoices.STARTED,
-                Snapshot.StatusChoices.PAUSED,
-            ],
+            status__in=Snapshot.OPEN_STATES,
         ).filter(
             Q(retry_at__isnull=True) | Q(retry_at__gt=now),
         ).update(
@@ -799,7 +906,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         )
         messages.success(request, f"Sealed {sealed} crawl(s). The runner will finish cleanup on the next sweep.")
 
-    @admin.action(description="Set Permissions ▾")
+    @admin.action(description="Permissions ▾")
     def set_crawl_permissions(self, request, queryset):
         permissions = (request.POST.get("permissions") or "").strip().lower()
         if permissions not in PERMISSIONS_VALUES:
@@ -812,11 +919,14 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         now = timezone.now()
         updated = 0
         batch = []
-        for crawl in queryset.only("id", "config").iterator(chunk_size=500):
+        crawls_to_update = []
+        for crawl in queryset.only("id", "config", "permissions").iterator(chunk_size=500):
+            old_permissions = crawl.permissions
             config = dict(crawl.config or {})
             config["PERMISSIONS"] = permissions
             crawl.config = config
             crawl.modified_at = now
+            crawls_to_update.append((crawl, old_permissions))
             batch.append(crawl)
             if len(batch) >= 500:
                 Crawl.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
@@ -825,9 +935,11 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         if batch:
             Crawl.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
             updated += len(batch)
+        for crawl, old_permissions in crawls_to_update:
+            crawl.update_child_snapshot_permissions(old_permissions, permissions)
         return updated
 
-    @action(label="Recrawl", description="Create a new crawl with the same settings")
+    @action(label="Recrawl", description="Create a new crawl with the same settings", methods=("POST",))
     def recrawl(self, request, obj):
         """Duplicate this crawl as a new crawl with the same URLs and settings."""
 
@@ -836,7 +948,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
             messages.error(request, "Cannot recrawl: original crawl has no URLs.")
             return redirect("admin:crawls_crawl_change", obj.id)
 
-        new_crawl = Crawl.objects.create(
+        new_crawl = Crawl.create_scheduler_row(
             urls=obj.urls,
             max_depth=obj.max_depth,
             tags_str=obj.tags_str,
@@ -861,47 +973,33 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         return format_html('<span class="crawl-stop-reason">{}</span>', reason)
 
     def stop_reason_for_crawl(self, obj):
-        from abx_dl.limits import CrawlLimitState
-
         if obj.pk in self.stop_reason_cache:
             return self.stop_reason_cache[obj.pk]
 
-        output_dir = obj.output_dir_for_config(self.crawl_admin_base_config)
+        output_dir = obj.output_dir
         config = self.limit_config_for_crawl(obj, output_dir)
-        reason = ""
-        if (output_dir / ".abx-dl" / "limits.json").exists():
-            config["CRAWL_DIR"] = str(output_dir)
-            reason = CrawlLimitState.from_config(config).get_stop_reason() or ""
-
-        max_urls = int(config["CRAWL_MAX_URLS"] or 0)
-        if not reason and max_urls > 0 and obj.num_snapshots_cached >= max_urls and obj.count_urls_for_limit() >= max_urls:
-            reason = "crawl_max_urls"
-
+        reason = obj.stop_reason(
+            config=config,
+            output_dir=output_dir,
+            num_snapshots=obj.num_snapshots_cached,
+            num_sealed_snapshots=obj.num_archived_snapshots_cached,
+        )
         self.stop_reason_cache[obj.pk] = reason
         return reason
 
     def limit_config_for_crawl(self, obj, output_dir):
-        config = {
-            "CRAWL_DIR": str(output_dir),
-            "CRAWL_MAX_URLS": self.crawl_admin_base_config.CRAWL_MAX_URLS,
-            "CRAWL_MAX_SIZE": self.crawl_admin_base_config.CRAWL_MAX_SIZE,
-            "CRAWL_TIMEOUT": self.crawl_admin_base_config.CRAWL_TIMEOUT,
-            "SNAPSHOT_MAX_SIZE": self.crawl_admin_base_config.SNAPSHOT_MAX_SIZE,
-        }
-        if obj.persona_id:
-            if obj.persona_id not in self.persona_limit_config_cache:
-                self.persona_limit_config_cache[obj.persona_id] = {
-                    key: value for key, value in obj.persona.get_derived_config().items() if key in config
-                }
-            config.update(self.persona_limit_config_cache[obj.persona_id])
-        if obj.config:
-            config.update({key: value for key, value in obj.config.items() if key in config})
-        return config
+        from archivebox.config.common import get_config
+
+        return get_config(crawl=obj).for_crawl_runtime(
+            crawl=obj,
+            persona=obj.resolve_persona(),
+            crawl_output_dir=output_dir,
+        )
 
     @admin.display(description="Status", ordering="status")
     def status_with_stop_reason(self, obj):
         status = "PAUSED" if obj.is_paused else str(obj.status or "").upper()
-        reason = self.stop_reason_for_crawl(obj) if obj.status == Crawl.StatusChoices.SEALED else ""
+        reason = self.stop_reason_for_crawl(obj) if obj.is_paused or obj.status == Crawl.StatusChoices.SEALED else ""
         if reason:
             reason_label = reason.removeprefix("crawl_").replace("_", " ")
             return format_html(
@@ -993,12 +1091,10 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Snapshots")
     def snapshots_changelist(self, obj):
-        request = getattr(self, "request", None)
+        request = self.request
         snapshot_changelist = reverse("admin:core_snapshot_changelist")
         scoped_params = {"crawl_id": str(obj.pk)}
         full_url = f"{snapshot_changelist}?{urlencode(scoped_params)}"
-        if request is None:
-            return format_html('<a class="button" href="{}">Open snapshots changelist</a>', full_url)
 
         snapshot_admin = self.admin_site._registry[Snapshot]
         changelist_request = copy(request)
@@ -1192,7 +1288,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
         return self.fieldsets
 
     def save_model(self, request, obj, form, change):
-        if not obj.created_by_id and getattr(request, "user", None) and request.user.is_authenticated:
+        if not obj.created_by_id and request.user.is_authenticated:
             obj.created_by = request.user
         super().save_model(request, obj, form, change)
 
@@ -1202,14 +1298,14 @@ class CrawlScheduleAdmin(BaseModelAdmin):
 
     @admin.display(description="# Crawls", ordering="crawl_count")
     def num_crawls(self, obj):
-        count = getattr(obj, "crawl_count", None)
+        count = obj.__dict__.get("crawl_count")
         if count is None:
             count = obj.crawl_set.count()
         return count
 
     @admin.display(description="# Snapshots", ordering="snapshot_count")
     def num_snapshots(self, obj):
-        count = getattr(obj, "snapshot_count", None)
+        count = obj.__dict__.get("snapshot_count")
         if count is None:
             count = Snapshot.objects.filter(crawl__schedule=obj).count()
         return count
@@ -1225,7 +1321,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
         crawl_ids = obj.crawl_set.values_list("pk", flat=True)
         return render_snapshots_list(
             Snapshot.objects.filter(crawl_id__in=crawl_ids),
-            request=getattr(self, "request", None),
+            request=self.request,
             prefix="schedule_snapshots",
         )
 

archivebox/crawls/migrations/0016_hydrate_crawl_permissions.py

@@ -1,5 +1,6 @@
 import os
 import json
+import uuid
 
 from django.db import migrations
 from django.db.models import Q
@@ -9,31 +10,6 @@ VALID_PERMISSIONS = {"public", "unlisted", "private"}
 BATCH_SIZE = 1000
 
 
-def legacy_bool(value):
-    if value is None:
-        return None
-    normalized = str(value).strip().lower()
-    if normalized in {"1", "true", "yes", "on"}:
-        return True
-    if normalized in {"0", "false", "no", "off"}:
-        return False
-    return None
-
-
-def permissions_from_legacy_public_flags(config):
-    if str(config.get("PERMISSIONS") or "").strip():
-        return None
-    public_snapshots = legacy_bool(config.get("PUBLIC_SNAPSHOTS"))
-    public_index = legacy_bool(config.get("PUBLIC_INDEX"))
-    if public_snapshots is False:
-        return "private"
-    if public_index is False:
-        return "unlisted"
-    if public_snapshots is True or public_index is True:
-        return "public"
-    return None
-
-
 def normalize_permissions(value, default):
     value = str(value or "").strip().lower()
     return value if value in VALID_PERMISSIONS else default
@@ -59,22 +35,19 @@ def raw_base_config(apps):
 
 
 def resolve_permissions(config, default):
+    from archivebox.config.common import permissions_from_legacy_public_flags
+
     explicit = str(config.get("PERMISSIONS") or "").strip().lower()
     if explicit in VALID_PERMISSIONS:
         return explicit
     return permissions_from_legacy_public_flags(config) or default
 
 
-def model_has_config(model):
-    try:
-        model._meta.get_field("config")
-    except Exception:
-        return False
-    return True
-
-
 def id_values(pk):
-    return str(pk), getattr(pk, "hex", str(pk).replace("-", ""))
+    if isinstance(pk, uuid.UUID):
+        return str(pk), pk.hex
+    pk_str = str(pk)
+    return pk_str, pk_str.replace("-", "")
 
 
 def flush_batch(cursor, table_name, batch):
@@ -125,8 +98,6 @@ def _ensure_permissions_column(cursor):
 
 def hydrate_crawl_permissions(apps, schema_editor):
     Crawl = apps.get_model("crawls", "Crawl")
-    User = apps.get_model("auth", "User")
-    user_has_config = model_has_config(User)
     base_config = raw_base_config(apps)
     default_permissions = resolve_permissions(base_config, "public")
     table_name = schema_editor.quote_name(Crawl._meta.db_table)
@@ -135,17 +106,13 @@ def hydrate_crawl_permissions(apps, schema_editor):
     batch = []
     missing_permissions = Q(permissions__isnull=True) | (Q(permissions__isnull=False) & ~Q(permissions__in=VALID_PERMISSIONS))
 
-    for crawl in Crawl.objects.filter(missing_permissions).select_related("persona", "created_by").iterator(chunk_size=BATCH_SIZE):
+    for crawl in Crawl.objects.filter(missing_permissions).select_related("persona").iterator(chunk_size=BATCH_SIZE):
         config = dict(crawl.config or {})
         resolved = dict(base_config)
         if crawl.persona_id:
             persona_config = crawl.persona.config or {}
             if isinstance(persona_config, dict):
                 resolved.update(persona_config)
-        if user_has_config:
-            user_config = crawl.created_by.config or {}
-            if isinstance(user_config, dict):
-                resolved.update(user_config)
         resolved.update(config)
         config["PERMISSIONS"] = resolve_permissions(resolved, default_permissions)
         batch.append((crawl.id, config))

archivebox/crawls/migrations/0018_freeze_crawl_config_snapshots.py

@@ -0,0 +1,67 @@
+import json
+
+from django.db import migrations
+
+
+BATCH_SIZE = 1000
+
+
+def _config_cache_key(config):
+    return json.dumps(config or {}, sort_keys=True, separators=(",", ":"), default=str)
+
+
+def _flush_updates(Crawl, db_alias, pending):
+    if pending:
+        Crawl.objects.using(db_alias).bulk_update(pending, ["config"], batch_size=BATCH_SIZE)
+        pending.clear()
+
+
+def freeze_existing_crawl_configs(apps, schema_editor):
+    from archivebox.config.common import build_crawl_config_snapshot
+    from archivebox.config.constants import CONSTANTS
+    from archivebox.personas.models import derive_persona_config
+
+    class PersonaConfigSnapshot:
+        def __init__(self, persona):
+            self.name = persona.name
+            self.config = dict(persona.config or {})
+
+        def get_derived_config(self):
+            return derive_persona_config(name=self.name, config=self.config, persona_dir=CONSTANTS.PERSONAS_DIR / self.name)
+
+    Crawl = apps.get_model("crawls", "Crawl")
+    Persona = apps.get_model("personas", "Persona")
+    db_alias = schema_editor.connection.alias
+    rows = Crawl.objects.using(db_alias).values_list("id", "persona_id", "config")
+    persona_ids = {persona_id for _, persona_id, _ in rows if persona_id}
+    personas = {persona.pk: PersonaConfigSnapshot(persona) for persona in Persona.objects.using(db_alias).filter(pk__in=persona_ids)}
+
+    frozen_cache = {}
+    pending = []
+    for crawl_id, persona_id, current_config in rows.iterator(chunk_size=BATCH_SIZE):
+        current_config = dict(current_config or {})
+        cache_key = (persona_id, _config_cache_key(current_config))
+        if cache_key not in frozen_cache:
+            frozen_cache[cache_key] = build_crawl_config_snapshot(
+                persona=personas.get(persona_id),
+                overrides=current_config,
+            )
+        frozen_config = frozen_cache[cache_key]
+        if frozen_config != current_config:
+            pending.append(Crawl(id=crawl_id, config=frozen_config))
+            if len(pending) >= BATCH_SIZE:
+                _flush_updates(Crawl, db_alias, pending)
+
+    _flush_updates(Crawl, db_alias, pending)
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("crawls", "0017_drop_stale_crawl_limit_columns"),
+    ]
+
+    operations = [
+        migrations.RunPython(freeze_existing_crawl_configs, migrations.RunPython.noop),
+    ]

archivebox/crawls/migrations/0019_crawlschedule_config.py

@@ -0,0 +1,25 @@
+from django.db import migrations, models
+
+
+def copy_template_config_to_schedule(apps, schema_editor):
+    CrawlSchedule = apps.get_model("crawls", "CrawlSchedule")
+    db_alias = schema_editor.connection.alias
+
+    for schedule in CrawlSchedule.objects.using(db_alias).select_related("template").iterator(chunk_size=200):
+        template_config = dict(schedule.template.config or {}) if schedule.template_id else {}
+        CrawlSchedule.objects.using(db_alias).filter(pk=schedule.pk).update(config=template_config)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("crawls", "0018_freeze_crawl_config_snapshots"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="crawlschedule",
+            name="config",
+            field=models.JSONField(blank=True, default=dict, null=True),
+        ),
+        migrations.RunPython(copy_template_config_to_schedule, migrations.RunPython.noop),
+    ]

archivebox/crawls/models.py

@@ -2,13 +2,12 @@ __package__ = "archivebox.crawls"
 
 from typing import TYPE_CHECKING, Any
 from collections.abc import Iterable, Mapping
-from io import StringIO
 import uuid
 import json
 import re
 from itertools import islice
 from datetime import timedelta
-from archivebox.uuid_compat import uuid7
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
 from urllib.parse import urlparse
 
@@ -35,14 +34,14 @@ from archivebox.base_models.models import (
 )
 from archivebox.workers.models import RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine
 from archivebox.crawls.schedule_util import next_run_for_schedule, validate_schedule
-from archivebox.misc.util import validate_url, validate_url_length
+from archivebox.misc.util import parse_date, sanitize_html_text, validate_url, validate_url_length
 
 if TYPE_CHECKING:
     from archivebox.core.models import Snapshot
 
 
 class CrawlSchedule(ModelWithUUID, ModelWithNotes):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     modified_at = models.DateTimeField(auto_now=True)
@@ -50,6 +49,7 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
     template: "Crawl" = models.ForeignKey("Crawl", on_delete=models.CASCADE, null=False, blank=False)  # type: ignore
     schedule = models.CharField(max_length=64, blank=False, null=False)
     is_enabled = models.BooleanField(default=True)
+    config = models.JSONField(default=dict, null=True, blank=True)
     label = models.CharField(max_length=64, blank=True, null=False, default="")
     notes = models.TextField(blank=True, null=False, default="")
 
@@ -69,9 +69,14 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
         return str(reverse_lazy("api-1:get_any", args=[self.id]))
 
     def save(self, *args, **kwargs):
+        update_fields = kwargs.get("update_fields")
+        if update_fields is None or "label" in update_fields:
+            self.label = sanitize_html_text(self.label).strip()
+        if update_fields is None or "notes" in update_fields:
+            self.notes = sanitize_html_text(self.notes)
         self.schedule = (self.schedule or "").strip()
         validate_schedule(self.schedule)
-        self.label = self.label or (self.template.label if self.template else "")
+        self.label = self.label or (sanitize_html_text(self.template.label).strip() if self.template else "")
         super().save(*args, **kwargs)
         if self.template:
             self.template.safe_update(
@@ -102,13 +107,16 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
         return self.is_enabled and self.next_run_at <= now
 
     def enqueue(self, queued_at=None) -> "Crawl":
+        from archivebox.config.common import build_crawl_config_snapshot
+
         queued_at = queued_at or timezone.now()
         template = self.template
         label = template.label or self.label
+        persona = template.persona if template.persona_id else None
 
         return Crawl.objects.create(
             urls=template.urls,
-            config=template.config or {},
+            config=build_crawl_config_snapshot(persona=persona, overrides=self.config or {}),
             max_depth=template.max_depth,
             tags_str=template.tags_str,
             persona_id=template.persona_id,
@@ -122,7 +130,7 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
 
 
 class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithStateMachine):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     modified_at = models.DateTimeField(auto_now=True)
@@ -162,12 +170,18 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     StatusChoices = ModelWithStateMachine.StatusChoices
     active_state = StatusChoices.STARTED
     delete_after_final_statuses = (StatusChoices.SEALED,)
+    RUNNABLE_STATES = (StatusChoices.QUEUED, StatusChoices.STARTED)
+    INACTIVE_STATES = (StatusChoices.PAUSED, StatusChoices.SEALED)
 
     schedule_id: uuid.UUID | None
-    sm: "CrawlMachine"
 
     snapshot_set: models.Manager["Snapshot"]
 
+    if TYPE_CHECKING:
+
+        @property
+        def sm(self) -> "CrawlMachine": ...
+
     class Meta(
         ModelWithDeleteAfter.Meta,
         ModelWithOutputDir.Meta,
@@ -190,9 +204,9 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         return f"[...{short_id}] {first_url[:120]}"
 
     def get_delete_after_config_value(self):
-        from archivebox.config.common import get_config
+        from archivebox.config.common import resolve_delete_after_config_value
 
-        return get_config(crawl=self).DELETE_AFTER
+        return resolve_delete_after_config_value(self.config)
 
     def pause(self, *, save: bool = True) -> bool:
         return super().pause(save=save)
@@ -238,11 +252,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         # child Snapshot through its own state machine. Active children that
         # are already due need no write; the runner will claim them as-is.
         active_children = self.snapshot_set.filter(
-            status__in=[
-                Snapshot.StatusChoices.QUEUED,
-                Snapshot.StatusChoices.STARTED,
-                Snapshot.StatusChoices.PAUSED,
-            ],
+            status__in=Snapshot.OPEN_STATES,
         )
         return active_children.filter(
             Q(retry_at__isnull=True) | Q(retry_at__gt=now),
@@ -259,10 +269,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         # Snapshot runner claim performs the real pause transition and cascades
         # its own ArchiveResults, keeping request/admin transactions tiny.
         active_children = self.snapshot_set.filter(
-            status__in=[
-                Snapshot.StatusChoices.QUEUED,
-                Snapshot.StatusChoices.STARTED,
-            ],
+            status__in=Snapshot.RUNNABLE_STATES,
         )
         return active_children.filter(
             Q(retry_at__isnull=True) | Q(retry_at__gt=now),
@@ -273,13 +280,16 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        from archivebox.personas.models import Persona
-
-        persona_ids = Persona.objects.filter(config__has_key="DELETE_AFTER").values_list("id", flat=True)
-        return cls.objects.filter(delete_at__isnull=True).filter(Q(config__has_key="DELETE_AFTER") | Q(persona_id__in=persona_ids))
+        return cls.objects.filter(delete_at__isnull=True, config__has_key="DELETE_AFTER")
 
     def save(self, *args, **kwargs):
         update_fields = kwargs.get("update_fields")
+        if update_fields is None or "label" in update_fields:
+            self.label = sanitize_html_text(self.label).strip()
+        if update_fields is None or "notes" in update_fields:
+            self.notes = sanitize_html_text(self.notes)
+        if update_fields is None or "tags_str" in update_fields:
+            self.tags_str = ",".join(self.parse_tag_names(self.tags_str or ""))
         sync_tags = update_fields is None or "tags_str" in update_fields
         old_crawl = type(self).objects.filter(pk=self.pk).first() if self.pk else None
         previous_tag_names = set()
@@ -287,12 +297,16 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             previous_tag_names = set(self.parse_tag_names(old_crawl.tags_str or ""))
 
         config = dict(self.config or {})
+        is_new = self._state.adding or old_crawl is None
+        persona = self.persona if self.persona_id else None
+        if is_new:
+            from archivebox.config.common import build_crawl_config_snapshot
+
+            config = build_crawl_config_snapshot(persona=persona, overrides=config)
         if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
             from archivebox.config.common import get_config
 
-            persona = self.persona if self.persona_id else None
-            user = self.created_by if self.created_by_id else None
-            config["PERMISSIONS"] = normalize_permissions(get_config(persona=persona, user=user, include_machine=True).PERMISSIONS)
+            config["PERMISSIONS"] = normalize_permissions(get_config(persona=persona, include_machine=True).PERMISSIONS)
         if "CRAWL_MAX_CONCURRENT_SNAPSHOTS" in config:
             raw_concurrency = config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]
             if raw_concurrency in (None, ""):
@@ -307,6 +321,9 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
 
         super().save(*args, **kwargs)
+        old_permissions = getattr(old_crawl, "permissions", None)
+        if old_crawl is not None and old_permissions != self.permissions:
+            transaction.on_commit(lambda: self.update_child_snapshot_permissions(old_permissions, self.permissions))
         if sync_tags:
             next_tag_names = set(self.parse_tag_names(self.tags_str or ""))
             added_tag_names = next_tag_names - previous_tag_names
@@ -336,9 +353,32 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         #         },
         #     )
 
+    def update_child_snapshot_permissions(self, old_permissions: str | None, new_permissions: str | None) -> int:
+        from archivebox.core.models import Snapshot
+
+        normalized_new_permissions = normalize_permissions(new_permissions)
+        now = timezone.now()
+        batch = []
+        updated = 0
+        queryset = self.snapshot_set.filter(Q(permissions=old_permissions) | Q(permissions__isnull=True)).only("id", "config")
+        for snapshot in queryset.iterator(chunk_size=500):
+            config = dict(snapshot.config or {})
+            config["PERMISSIONS"] = normalized_new_permissions
+            snapshot.config = config
+            snapshot.modified_at = now
+            batch.append(snapshot)
+            if len(batch) >= 500:
+                Snapshot.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
+                updated += len(batch)
+                batch.clear()
+        if batch:
+            Snapshot.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
+            updated += len(batch)
+        return updated
+
     @property
     def api_url(self) -> str:
-        return str(reverse_lazy("api-1:get_crawl", args=[self.id.hex]))
+        return str(reverse_lazy("api-1:get_crawl", args=[self.id]))
 
     @staticmethod
     def parse_tag_names(tags: Iterable[str] | str, *, pattern: str = r",") -> list[str]:
@@ -346,7 +386,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         tag_names: list[str] = []
         seen: set[str] = set()
         for raw_tag in raw_tags:
-            tag_name = str(raw_tag or "").strip()
+            tag_name = sanitize_html_text(raw_tag).strip()
             if not tag_name:
                 continue
             lowered = tag_name.lower()
@@ -467,37 +507,36 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         )
         return crawl
 
-    def output_dir_for_config(self, runtime_config: Mapping[str, Any] | Any) -> Path:
-        """
-        Construct output directory: archive/users/{username}/crawls/{YYYYMMDD}/{domain}/{crawl-id}
-        Domain is extracted from the first URL in the crawl.
-        """
+    @property
+    def output_dir(self) -> Path:
         from archivebox.config import CONSTANTS
         from archivebox.core.models import Snapshot
 
         date_str = self.created_at.strftime("%Y%m%d")
-        first_url = ""
-        for raw_line in StringIO(self.urls or ""):
-            candidate = raw_line.strip()
-            if candidate and not candidate.startswith("#"):
-                first_url = candidate
-                break
+        first_url = next((url for url in self.get_urls_list() if url), "")
         domain = Snapshot.extract_domain_from_url(first_url) if first_url else "unknown"
 
-        users_dir = Path(runtime_config["USERS_DIR"]) if isinstance(runtime_config, Mapping) else runtime_config.USERS_DIR
-        return users_dir / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id)
-
-    @property
-    def output_dir(self) -> Path:
-        from archivebox.config.common import get_config
-
-        return self.output_dir_for_config(get_config(resolve_plugins=False))
+        output_dir = CONSTANTS.USERS_DIR / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id)
+        hyphen_dir = output_dir.with_name(str(uuid.UUID(hex=self.id.hex)))
+        return output_dir if output_dir.exists() or not hyphen_dir.exists() else hyphen_dir
 
     def get_urls_list(self) -> list[str]:
         """Get list of URLs from urls field, filtering out comments and empty lines."""
         if not self.urls:
             return []
-        return [url.strip() for url in self.urls.split("\n") if url.strip() and not url.strip().startswith("#")]
+        return [url for _raw_line, url in self._iter_url_lines() if url]
+
+    def has_internal_input_root(self) -> bool:
+        """Return True when Crawl.urls is preserved source text, not the work queue.
+
+        The runner creates a synthetic root snapshot for raw import text so
+        parser hooks use the same Snapshot lifecycle as every other extractor.
+        In that mode the submitted text must remain in Crawl.urls forever;
+        parsed URLs live as child Snapshot rows and should not be appended back.
+        """
+        from archivebox.core.models import Snapshot
+
+        return self.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL, depth=0).exists()
 
     @staticmethod
     def normalize_domain(value: str) -> str:
@@ -566,22 +605,29 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         except re.error:
             return False
 
+    def get_current_config(self, *, refresh: bool = False) -> dict[str, Any]:
+        if refresh and self.pk:
+            config = type(self).objects.filter(pk=self.pk).values_list("config", flat=True).first()
+            if config is not None:
+                self.config = config
+        return dict(self.config or {})
+
     def get_url_allowlist(self, *, use_effective_config: bool = False, snapshot=None) -> list[str]:
         if use_effective_config:
-            from archivebox.config.common import get_config
-
-            config = get_config(crawl=self, snapshot=snapshot)
+            config = self.get_current_config(refresh=True)
         else:
-            config = self.config or {}
+            config = self.get_current_config()
+        if snapshot is not None and snapshot.config:
+            config.update(snapshot.config)
         return self.split_filter_patterns(config.get("URL_ALLOWLIST", ""))
 
     def get_url_denylist(self, *, use_effective_config: bool = False, snapshot=None) -> list[str]:
         if use_effective_config:
-            from archivebox.config.common import get_config
-
-            config = get_config(crawl=self, snapshot=snapshot)
+            config = self.get_current_config(refresh=True)
         else:
-            config = self.config or {}
+            config = self.get_current_config()
+        if snapshot is not None and snapshot.config:
+            config.update(snapshot.config)
         return self.split_filter_patterns(config.get("URL_DENYLIST", ""))
 
     def url_passes_filters(self, url: str, *, snapshot=None, use_effective_config: bool = True) -> bool:
@@ -660,7 +706,14 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 continue
             try:
                 entry = json.loads(stripped)
-                entries.append((raw_line.rstrip(), str(entry.get("url", "") or "").strip()))
+                # Crawl.urls accepts plain lines and JSONL URL records. Other
+                # valid JSON values, e.g. a quoted string from hostile input,
+                # are not records and must stay inert text instead of raising
+                # during later Snapshot.save() bookkeeping.
+                if isinstance(entry, dict):
+                    entries.append((raw_line.rstrip(), str(entry.get("url", "") or "").strip()))
+                else:
+                    entries.append((raw_line.rstrip(), stripped))
             except json.JSONDecodeError:
                 entries.append((raw_line.rstrip(), stripped))
         return entries
@@ -682,9 +735,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         return len(urls)
 
     def remaining_url_capacity(self) -> int | None:
-        from archivebox.config.common import get_config
-
-        max_urls = int(get_config(crawl=self).CRAWL_MAX_URLS or 0)
+        max_urls = int(self._config_value(self.get_current_config(refresh=True), "CRAWL_MAX_URLS", 0) or 0)
         if max_urls <= 0:
             return None
         return max(max_urls - self.count_urls_for_limit(), 0)
@@ -694,9 +745,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         return remaining is None or remaining > 0
 
     def remaining_snapshot_capacity(self) -> int | None:
-        from archivebox.config.common import get_config
-
-        max_urls = int(get_config(crawl=self).CRAWL_MAX_URLS or 0)
+        max_urls = int(self._config_value(self.get_current_config(refresh=True), "CRAWL_MAX_URLS", 0) or 0)
         if max_urls <= 0:
             return None
         return max(max_urls - self.snapshot_set.count(), 0)
@@ -771,35 +820,105 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         from archivebox.personas.models import Persona
 
         if self.persona_id:
-            persona = Persona.objects.filter(id=self.persona_id).first()
-            if persona is not None:
-                return persona
-
-        default_persona_name = str((self.config or {}).get("DEFAULT_PERSONA") or "").strip()
-        if default_persona_name:
-            persona, _ = Persona.objects.get_or_create(name=default_persona_name or "Default")
-            persona.ensure_dirs()
-            return persona
+            return Persona.objects.filter(id=self.persona_id).first()
 
         return None
 
-    def limit_stop_reason(self) -> str:
-        from abx_dl.limits import CrawlLimitState
-        from archivebox.config.common import get_config
+    @staticmethod
+    def _config_value(config: Mapping[str, Any] | Any, key: str, default: Any = None) -> Any:
+        if isinstance(config, Mapping):
+            return config.get(key, default)
+        return config[key] if key in config else default
 
-        config = get_config(crawl=self, include_machine=False)
-        if (self.output_dir / ".abx-dl" / "limits.json").exists():
-            config["CRAWL_DIR"] = str(self.output_dir)
+    @classmethod
+    def create_scheduler_row(cls, **kwargs) -> "Crawl":
+        from archivebox.base_models.models import normalize_config_json_values
+        from archivebox.config.common import build_crawl_config_snapshot
+
+        now = timezone.now()
+        kwargs.setdefault("created_at", now)
+        kwargs.setdefault("modified_at", now)
+        config = normalize_config_json_values(kwargs.get("config") or {})
+        persona = kwargs.get("persona")
+        if persona is None and kwargs.get("persona_id"):
+            from archivebox.personas.models import Persona
+
+            persona = Persona.objects.filter(pk=kwargs["persona_id"]).first()
+        kwargs["config"] = build_crawl_config_snapshot(persona=persona, overrides=config)
+        crawl = cls(**kwargs)
+        if crawl.delete_at is None:
+            crawl.set_delete_at_from_config()
+        cls.objects.bulk_create([crawl])
+        return crawl
+
+    def limit_stop_reason(
+        self,
+        *,
+        config: Mapping[str, Any] | Any | None = None,
+        output_dir: Path | None = None,
+        num_snapshots: int | None = None,
+    ) -> str:
+        from abx_dl.limits import CrawlLimitState
+
+        if output_dir is None:
+            output_dir = self.output_dir
+        if config is None:
+            from archivebox.config.common import get_config
+
+            config = get_config(crawl=self, include_machine=False).for_crawl_runtime(
+                crawl=self,
+                persona=self.resolve_persona(),
+                crawl_output_dir=output_dir,
+            )
+
+        limits_path = output_dir / ".abx-dl" / "limits.json"
+        if limits_path.exists():
             stop_reason = CrawlLimitState.from_config(config).get_stop_reason()
             if stop_reason:
                 return stop_reason
 
-        max_urls = int(config.CRAWL_MAX_URLS or 0)
-        if max_urls > 0 and self.snapshot_set.count() >= max_urls and self.count_urls_for_limit() >= max_urls:
+        max_urls = int(self._config_value(config, "CRAWL_MAX_URLS", 0) or 0)
+        if num_snapshots is None:
+            num_snapshots = self.snapshot_set.count()
+        if max_urls > 0 and num_snapshots >= max_urls and self.count_urls_for_limit() >= max_urls:
             return "crawl_max_urls"
 
         return ""
 
+    def lifecycle_stop_reason(self, *, num_snapshots: int | None = None, num_sealed_snapshots: int | None = None) -> str:
+        if self.is_paused:
+            return "paused"
+
+        if self.status != self.StatusChoices.SEALED:
+            return ""
+
+        if num_snapshots is None:
+            num_snapshots = self.snapshot_set.count()
+        if num_snapshots == 0:
+            return "no_viable_urls"
+
+        if num_sealed_snapshots is None:
+            from archivebox.core.models import Snapshot
+
+            num_sealed_snapshots = self.snapshot_set.filter(status=Snapshot.StatusChoices.SEALED).count()
+        if num_sealed_snapshots >= num_snapshots:
+            return "done"
+
+        return ""
+
+    def stop_reason(
+        self,
+        *,
+        config: Mapping[str, Any] | Any | None = None,
+        output_dir: Path | None = None,
+        num_snapshots: int | None = None,
+        num_sealed_snapshots: int | None = None,
+    ) -> str:
+        return self.limit_stop_reason(config=config, output_dir=output_dir, num_snapshots=num_snapshots) or self.lifecycle_stop_reason(
+            num_snapshots=num_snapshots,
+            num_sealed_snapshots=num_sealed_snapshots,
+        )
+
     def add_url(self, entry: dict) -> bool:
         """
         Add a URL to the crawl queue if not already present.
@@ -856,21 +975,27 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             List of newly created Snapshot objects
         """
         from archivebox.core.models import Snapshot, Tag
-        from archivebox.config.common import get_config
         from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
 
         if self.status == self.StatusChoices.SEALED:
             return []
+        # Internal-input crawls preserve the submitted text verbatim in
+        # Crawl.urls. The root snapshot's parser hooks are the only supported
+        # path for turning that text into child snapshots, otherwise a later
+        # runner pass could reinterpret plain URL-looking lines as direct
+        # depth-0 work and bypass format-specific metadata parsing.
+        if self.has_internal_input_root():
+            return []
 
         created_snapshots = []
         crawl_tag_names = self.current_tag_names()
         tags_by_name: dict[str, Tag] = {}
-        config = get_config(crawl=self)
-        only_new_urls = bool(config.ONLY_NEW)
 
         for line in self.urls.splitlines():
             if not line.strip():
                 continue
+            config = self.get_current_config(refresh=True)
+            only_new_urls = bool(self._config_value(config, "ONLY_NEW", True))
 
             # Parse JSONL or plain URL
             try:
@@ -896,10 +1021,10 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             except ValueError as err:
                 print(f"[yellow][!] Skipping invalid snapshot URL: {url[:120]}... ({err})[/yellow]")
                 continue
-            if Snapshot.is_archivebox_internal_url(url):
+            if Snapshot.is_archivebox_internal_url(url, config=config):
                 print(f"[yellow][!] Skipping internal ArchiveBox snapshot URL: {url}[/yellow]")
                 continue
-            if not self.url_passes_filters(url):
+            if not self.url_passes_filters(url, use_effective_config=False):
                 continue
             if only_new_urls and Snapshot.objects.filter(url=url).exists():
                 continue
@@ -969,7 +1094,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                         # relation without clearing any non-crawl snapshot tags.
                         Tag.objects.bulk_create(missing_tags, ignore_conflicts=True)
                         tags_by_name.update({tag.name: tag for tag in Tag.objects.filter(name__in=missing_names)})
-                snapshot.tags.add(*[tag.pk for tag_name in tag_names if (tag := tags_by_name.get(tag_name))])
+                snapshot.add_tag_ids([tag.pk for tag_name in tag_names if (tag := tags_by_name.get(tag_name))])
 
             # Symlink creation touches the filesystem and can be slow on remote disks.
             # Defer it until after any active DB transaction commits so SQLite does
@@ -1007,7 +1132,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     ) -> list["Snapshot"]:
         """Create child snapshots from discovered URL records after filtering and deduping once."""
         from archivebox.core.models import Snapshot, SnapshotTag, Tag
-        from archivebox.config.common import get_config
         from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
 
         if self.status == self.StatusChoices.SEALED:
@@ -1017,30 +1141,71 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             return []
 
         crawl_tag_names = self.current_tag_names()
-        config = get_config(crawl=self, snapshot=parent_snapshot)
+        config = self.get_current_config(refresh=True)
+        if parent_snapshot is not None and parent_snapshot.config:
+            config.update(parent_snapshot.config)
         allowlist = self.split_filter_patterns(config.get("URL_ALLOWLIST", ""))
         denylist = self.split_filter_patterns(config.get("URL_DENYLIST", ""))
 
+        def metadata_score(record: Mapping[str, Any]) -> int:
+            # Multiple parsers can discover the same URL from one import root.
+            # Keep the record with the richest user-facing metadata so generic
+            # text/HTML extraction does not erase RSS/Netscape/JSON fields.
+            return sum(bool(record.get(field)) for field in ("title", "bookmarked_at", "timestamp", "tags"))
+
         deduped_records: dict[str, Mapping[str, Any]] = {}
         for record in records:
             url = sanitize_extracted_url(fix_url_from_markdown(str(record.get("url") or "").strip()))
-            if not url or url in deduped_records:
+            if not url:
                 continue
             try:
                 validate_url(url)
             except ValueError as err:
                 print(f"[yellow][!] Skipping invalid discovered snapshot URL: {url[:120]}... ({err})[/yellow]")
                 continue
-            if Snapshot.is_archivebox_internal_url(url):
+            if Snapshot.is_archivebox_internal_url(url, config=config):
                 print(f"[yellow][!] Skipping internal ArchiveBox discovered snapshot URL: {url}[/yellow]")
                 continue
             if self.url_passes_compiled_filters(url, allowlist=allowlist, denylist=denylist):
-                deduped_records[url] = record
+                existing_record = deduped_records.get(url)
+                if existing_record is None or metadata_score(record) > metadata_score(existing_record):
+                    deduped_records[url] = record
 
         if not deduped_records:
             return []
 
-        existing_scope = Snapshot.objects if bool(config.ONLY_NEW) else self.snapshot_set
+        existing_in_crawl = {
+            snapshot.url: snapshot for snapshot in self.snapshot_set.prefetch_related("tags").filter(url__in=deduped_records.keys())
+        }
+        for url, snapshot in existing_in_crawl.items():
+            record = deduped_records[url]
+            update_fields = []
+            title = Snapshot._normalize_title_candidate(str(record.get("title") or "").strip()[:512], snapshot_url=url)
+            if title and (not snapshot.title or len(title) > len(snapshot.title or "")):
+                snapshot.title = title
+                update_fields.append("title")
+            bookmarked_at = None
+            try:
+                bookmarked_at = parse_date(record.get("bookmarked_at") or record.get("timestamp"))
+            except (TypeError, ValueError, OSError):
+                pass
+            if bookmarked_at and snapshot.bookmarked_at != bookmarked_at:
+                snapshot.bookmarked_at = bookmarked_at
+                update_fields.append("bookmarked_at")
+            if update_fields:
+                snapshot.save(update_fields=[*update_fields, "modified_at"])
+            tag_names = {
+                *crawl_tag_names,
+                *self.parse_tag_names(
+                    str(record.get("tags") or ""),
+                    pattern=self._config_value(config, "TAG_SEPARATOR_PATTERN", r"[,]"),
+                ),
+            }
+            if tag_names:
+                tag_ids = [Tag.objects.get_or_create(name=tag_name)[0].pk for tag_name in tag_names]
+                snapshot.add_tag_ids(tag_ids)
+
+        existing_scope = Snapshot.objects if bool(self._config_value(config, "ONLY_NEW", True)) else self.snapshot_set
         existing_urls = set(existing_scope.filter(url__in=deduped_records.keys()).values_list("url", flat=True))
         urls = [url for url in deduped_records.keys() if url not in existing_urls]
         remaining = self.remaining_snapshot_capacity()
@@ -1050,27 +1215,34 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             return []
 
         now = timezone.now()
-        snapshots = [
-            Snapshot(
-                url=url,
-                timestamp=str((now + timedelta(microseconds=index)).timestamp()),
-                title=Snapshot._normalize_title_candidate(
-                    str(deduped_records[url].get("title") or "").strip()[:512],
-                    snapshot_url=url,
-                )
-                or None,
-                crawl=self,
-                parent_snapshot=parent_snapshot,
-                depth=depth,
-                status=Snapshot.StatusChoices.QUEUED,
-                retry_at=now,
-                bookmarked_at=now,
-                created_at=now,
+        snapshots = []
+        for index, url in enumerate(urls):
+            record = deduped_records[url]
+            bookmarked_at = now
+            try:
+                bookmarked_at = parse_date(record.get("bookmarked_at") or record.get("timestamp")) or now
+            except (TypeError, ValueError, OSError):
+                pass
+            snapshots.append(
+                Snapshot(
+                    url=url,
+                    timestamp=str((now + timedelta(microseconds=index)).timestamp()),
+                    title=Snapshot._normalize_title_candidate(
+                        str(record.get("title") or "").strip()[:512],
+                        snapshot_url=url,
+                    )
+                    or None,
+                    crawl=self,
+                    parent_snapshot=parent_snapshot,
+                    depth=depth,
+                    status=Snapshot.StatusChoices.QUEUED,
+                    retry_at=now,
+                    bookmarked_at=bookmarked_at,
+                    created_at=now,
+                ),
             )
-            for index, url in enumerate(urls)
-        ]
         for snapshot in snapshots:
-            snapshot.set_delete_at_from_config(config.DELETE_AFTER)
+            snapshot.set_delete_at_from_config(self._config_value(config, "DELETE_AFTER", "0"))
 
         created_snapshots = []
         for snapshot in snapshots:
@@ -1089,7 +1261,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         crawl_urls = {url for _raw_line, url in self._iter_url_lines() if url}
         new_url_lines = [snapshot.url for snapshot in created_snapshots if snapshot.url not in crawl_urls]
-        if new_url_lines:
+        # For internal-input crawls, Crawl.urls is the immutable source text.
+        # Child snapshots are the parsed/indexed representation, so appending
+        # discovered URLs here would both duplicate state and destroy the exact
+        # import artifact users submitted through CLI/API/UI.
+        if new_url_lines and not self.has_internal_input_root():
             self.urls = (self.urls.rstrip() + "\n" + "\n".join(new_url_lines)).lstrip("\n")
             self.save(update_fields=["urls", "modified_at"])
 
@@ -1097,7 +1273,10 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         for snapshot in created_snapshots:
             tag_names = {
                 *crawl_tag_names,
-                *self.parse_tag_names(str(deduped_records[snapshot.url].get("tags") or ""), pattern=config.TAG_SEPARATOR_PATTERN),
+                *self.parse_tag_names(
+                    str(deduped_records[snapshot.url].get("tags") or ""),
+                    pattern=self._config_value(config, "TAG_SEPARATOR_PATTERN", r"[,]"),
+                ),
             }
             if tag_names:
                 tag_names_by_url[snapshot.url] = tag_names
@@ -1220,15 +1399,16 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             The root Snapshot for this crawl, or None for system crawls that don't create snapshots
         """
         import time
-        from archivebox.hooks import run_hook, discover_hooks, process_hook_records, is_finite_background_hook
+        from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records
         from archivebox.config.common import get_config
         from archivebox.machine.models import Binary, Machine
 
         def get_runtime_config():
-            config = get_config(crawl=self)
-            if persona_runtime_overrides:
-                config.update(persona_runtime_overrides)
-            return config
+            return get_config(crawl=self).for_crawl_runtime(
+                crawl=self,
+                persona=persona,
+                runtime_overrides=persona_runtime_overrides,
+            )
 
         system_task = self.get_system_task()
         if system_task == "archivebox://update":
@@ -1277,19 +1457,15 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 print(f"[yellow]⏱️  Hook {hook.name} took {hook_elapsed:.2f}s[/yellow]")
 
             if process.status == process.StatusChoices.RUNNING:
-                if not is_finite_background_hook(hook.name):
-                    return set()
-                try:
-                    process.wait(timeout=process.timeout)
-                except Exception:
+                if process.poll() is None:
                     return set()
 
-            from archivebox.hooks import extract_records_from_process
+            from archivebox.plugins.hooks import extract_records_from_process
 
             records = []
-            # Finite background hooks can exit before their completed Process
-            # metadata is visible. Give successful hooks a brief chance to
-            # flush JSONL stdout into the Process row before downstream hooks.
+            # A hook can exit before its completed Process metadata is visible.
+            # Give successful hooks a brief chance to flush JSONL stdout into
+            # the Process row before downstream hooks.
             for delay in (0.0, 0.05, 0.1, 0.25, 0.5):
                 if delay:
                     time.sleep(delay)
@@ -1409,7 +1585,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     def cleanup(self):
         """Clean up background hooks and run on_CrawlEnd hooks."""
-        from archivebox.hooks import run_hook, discover_hooks
+        from archivebox.plugins.hooks import run_hook, discover_hooks
 
         # Clean up .pid files from output directory
         if self.output_dir.exists():