release: archivebox 0.9.35rc15

Updated copyright year in the LICENSE file.

.claude/settings.json

@@ -1,5 +0,0 @@
-{
-  "enabledPlugins": {
-    "pyright-lsp@claude-plugins-official": true
-  }
-}

.claude/settings.local.json

@@ -1,51 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Read(**)",
-      "Glob(**)",
-      "Grep(**)",
-      "Bash(python -m archivebox:*)",
-      "Bash(ls:*)",
-      "Bash(xargs:*)",
-      "Bash(python -c:*)",
-      "Bash(printf:*)",
-      "Bash(pkill:*)",
-      "Bash(python3:*)",
-      "Bash(sqlite3:*)",
-      "WebFetch(domain:github.com)",
-      "Bash(uv add:*)",
-      "Bash(mkdir:*)",
-      "Bash(chmod:*)",
-      "Bash(python -m forum_dl:*)",
-      "Bash(archivebox manage migrate:*)",
-      "Bash(cat:*)",
-      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
-      "Bash(forum-dl:*)",
-      "Bash(pip uninstall:*)",
-      "Bash(python:*)",
-      "Bash(source .venv/bin/activate)",
-      "Bash(mv:*)",
-      "Bash(echo:*)",
-      "Bash(grep:*)",
-      "WebFetch(domain:python-statemachine.readthedocs.io)",
-      "Bash(./bin/run_plugin_tests.sh:*)",
-      "Bash(done)",
-      "Bash(coverage erase:*)",
-      "Bash(gh api:*)"
-    ]
-  },
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Bash",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
-            "statusMessage": "Checking working directory..."
-          }
-        ]
-      }
-    ]
-  }
-}

.dockerignore

@@ -3,18 +3,29 @@
 /.heartbeat.json
 /package.json
 /package-lock.json
+__pycache__
+**/__pycache__
 *.pyc
-__pycache__/
+*.pyo
+*.py[cod]
+*$py.class
 .mypy_cache/
 .pytest_cache/
+.ruff_cache/
+.uv-cache/
 .github/
 .pdm-build/
 .pdm-python
 .eggs/
 .git/
+!.git/
+.git/*
 .vscode/
 !.git/HEAD
-!.git/refs/heads/*
+!.git/packed-refs
+!.git/refs/
+!.git/refs/heads/
+!.git/refs/heads/**
 
 venv/
 .venv/
@@ -38,6 +49,8 @@ snapshots/
 logs/
 archivebox-docker-smoke*/
 archivebox-compose-smoke*/
+docker-test/
+docker-test*/
 core
 *.core
 

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -69,7 +69,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:

.github/ISSUE_TEMPLATE/2-feature_request.yml

@@ -71,7 +71,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:

.github/workflows/codeql.yml

@@ -19,6 +19,10 @@ on:
   schedule:
     - cron: '33 17 * * 6'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   analyze:
     name: Analyze (${{ matrix.language }})

.github/workflows/debian.yml

@@ -1,210 +0,0 @@
-name: Build Debian package
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'pkg/debian/**'
-      - 'bin/build_deb.sh'
-      - 'bin/release_deb.sh'
-      - '.github/workflows/debian.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: write
-
-jobs:
-  build:
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        arch: [amd64, arm64]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Install nfpm
-        env:
-          NFPM_VERSION: "2.45.1"
-        run: |
-          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \
-            | sudo tar -xz -C /usr/local/bin nfpm
-          nfpm --version
-
-      - name: Build .deb package
-        run: |
-          export VERSION="${{ steps.version.outputs.version }}"
-          export ARCH="${{ matrix.arch }}"
-          ./bin/build_deb.sh
-
-      - name: Verify .deb package contents
-        run: |
-          DEB_FILE="$(ls dist/archivebox*.deb | head -1)"
-          echo "=== Package info ==="
-          dpkg-deb --info "$DEB_FILE"
-          echo ""
-          echo "=== Package contents ==="
-          dpkg-deb --contents "$DEB_FILE"
-          echo ""
-          echo "=== Control fields ==="
-          dpkg-deb --field "$DEB_FILE"
-
-      - name: Upload .deb artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb
-          path: dist/*.deb
-
-  test:
-    needs: build
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-            runner: ubuntu-24.04
-          - arch: arm64
-            runner: ubuntu-24.04-arm
-    runs-on: ${{ matrix.runner }}
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local wheel
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --wheel --out-dir /tmp/wheels/
-
-      - name: Download .deb artifact
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*-${{ matrix.arch }}.deb
-          merge-multiple: true
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget
-
-      - name: Pre-seed virtualenv with local wheel before dpkg install
-        run: |
-          # CI-only: pre-seed the venv with a local wheel so we test the
-          # unreleased code, not whatever is on PyPI. We explicitly use python3.13
-          # here (matching the system dep installed above) to ensure the venv is
-          # created with the correct Python version. On real installs, install.sh
-          # handles this by preferring python3.13 and failing if < 3.13.
-          # When postinstall.sh runs during dpkg -i, it finds the venv already
-          # populated and just upgrades deps.
-          sudo mkdir -p /opt/archivebox
-          sudo python3.13 -m venv /opt/archivebox/venv
-          sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools
-          sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl
-          echo "[√] Pre-seeded /opt/archivebox/venv with local wheel"
-
-      - name: Install .deb package
-        run: |
-          # dpkg install will run postinstall which creates the user,
-          # sets up systemd, and tries pip install (which finds it already installed)
-          sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y
-
-      - name: Verify archivebox is installed
-        run: |
-          which archivebox
-          archivebox version
-
-      - name: Verify wrapper script works
-        run: |
-          /usr/bin/archivebox version
-          /usr/bin/archivebox --help | head -5
-
-      - name: Test archivebox init as archivebox user
-        run: |
-          # The postinstall should have created the user
-          id archivebox
-          sudo mkdir -p /tmp/archivebox-test
-          sudo chown archivebox:archivebox /tmp/archivebox-test
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init'
-
-      - name: Test archivebox status
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status'
-
-      - name: Test archivebox add
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"'
-
-      - name: Verify systemd service file exists
-        run: |
-          test -f /usr/lib/systemd/system/archivebox.service
-          cat /usr/lib/systemd/system/archivebox.service
-
-  # Upload .deb to GitHub Release only after tests pass
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: [build, test]
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: write
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Determine release tag
-        id: tag
-        run: |
-          TAG="${{ github.event.release.tag_name }}"
-          if [ -z "$TAG" ]; then
-            TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')"
-            echo "[i] No release tag in event context, using version from pyproject.toml: $TAG"
-          fi
-          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
-
-      - name: Download all .deb artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*.deb
-          merge-multiple: true
-
-      - name: Upload .deb to GitHub Release
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          TAG="${{ steps.tag.outputs.tag }}"
-          # Verify the release exists before uploading
-          if ! gh release view "$TAG" >/dev/null 2>&1; then
-            echo "[!] No GitHub Release found for tag $TAG."
-            if [ -n "${{ github.event.release.tag_name }}" ]; then
-              echo "[X] This was triggered by a release event — the release should exist. Failing."
-              exit 1
-            fi
-            echo "[i] Skipping upload (workflow_dispatch without a release)."
-            echo "    Create a release first or trigger via the release event."
-            exit 0
-          fi
-          gh release upload "$TAG" *.deb --clobber

.github/workflows/docker.yml

@@ -5,7 +5,7 @@ on:
   workflow_call:
   push:
     branches:
-      - dev
+      - "**"
     tags:
       - 'v*'
   # pull_request:
@@ -13,13 +13,14 @@ on:
 env:
   DOCKERHUB_IMAGE: archivebox/archivebox
   GHCR_IMAGE: ghcr.io/archivebox/archivebox
+  ABX_DL_IMAGE: archivebox/abx-dl:latest
 
 permissions:
   contents: read
   packages: write
 
 concurrency:
-  group: docker-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -45,6 +46,44 @@ jobs:
         #   submodules: true
         #   fetch-depth: 1
 
+      - name: Wait for released ArchiveBox deps on PyPI
+        run: |
+          python3 - <<'PY'
+          import json
+          import re
+          import sys
+          import time
+          import tomllib
+          import urllib.request
+
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp:
+                      releases = json.load(resp)["releases"]
+                  if version in releases:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
+          PY
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
@@ -58,6 +97,24 @@ jobs:
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
+      - name: Wait for published abx-dl image
+        id: abx_dl_image
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          deadline=$((SECONDS + 1800))
+          until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
+            if (( SECONDS >= deadline )); then
+              echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2
+              exit 1
+            fi
+            echo "${ABX_DL_IMAGE} is not published yet; waiting..."
+            sleep 30
+          done
+
+          echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
+          docker buildx imagetools inspect "${ABX_DL_IMAGE}"
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         if: github.event_name != 'pull_request'
@@ -78,13 +135,10 @@ jobs:
         run: |
           set -Eeuo pipefail
           VERSION="$(python3 - <<'PY'
-          from pathlib import Path
-          import re
+          import tomllib
 
-          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
-          if not match:
-              raise SystemExit("Failed to read version from pyproject.toml")
-          print(match.group(1))
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
           PY
           )"
 
@@ -93,11 +147,12 @@ jobs:
             echo "org.opencontainers.image.version=${VERSION}"
             echo "org.opencontainers.image.revision=${GITHUB_SHA}"
             echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
+            echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
             echo 'EOF'
             echo "version=${VERSION}"
           } >> "$GITHUB_OUTPUT"
 
-          echo "[+] Building ${{ matrix.platform }} for ${VERSION}"
+          echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
 
       - name: Build and push digest
         id: docker_build
@@ -111,6 +166,8 @@ jobs:
             ${{ env.DOCKERHUB_IMAGE }}
             ${{ env.GHCR_IMAGE }}
           labels: ${{ steps.docker_meta.outputs.labels }}
+          build-args: |
+            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
           cache-from: type=gha,scope=${{ matrix.cache_scope }}
           cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
           platforms: ${{ matrix.platform }}
@@ -177,36 +234,49 @@ jobs:
         run: |
           set -Eeuo pipefail
           VERSION="$(python3 - <<'PY'
-          from pathlib import Path
-          import re
+          import tomllib
 
-          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
-          if not match:
-              raise SystemExit("Failed to read version from pyproject.toml")
-          print(match.group(1))
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
           PY
           )"
-          SHORT_SHA="${GITHUB_SHA::8}"
+          BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)"
+          SHORT_SHA="${GITHUB_SHA::12}"
+          test -n "$BRANCH_TAG"
+          test -n "$SHORT_SHA"
 
           {
             echo 'dockerhub_tags<<EOF'
-            echo "${DOCKERHUB_IMAGE}:dev"
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${DOCKERHUB_IMAGE}:latest"
+            fi
+            echo "${DOCKERHUB_IMAGE}:${BRANCH_TAG}"
             echo "${DOCKERHUB_IMAGE}:${VERSION}"
             echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
             echo 'EOF'
             echo 'ghcr_tags<<EOF'
-            echo "${GHCR_IMAGE}:dev"
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${GHCR_IMAGE}:latest"
+            fi
+            echo "${GHCR_IMAGE}:${BRANCH_TAG}"
             echo "${GHCR_IMAGE}:${VERSION}"
             echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
             echo 'EOF'
             echo "version=${VERSION}"
+            echo "branch_tag=${BRANCH_TAG}"
             echo "short_sha=${SHORT_SHA}"
           } >> "$GITHUB_OUTPUT"
 
           echo "[+] Publishing Docker Hub tags:"
-          printf '%s\n' "${DOCKERHUB_IMAGE}:dev" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${DOCKERHUB_IMAGE}:latest"
+          fi
+          printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
           echo "[+] Publishing GHCR tags:"
-          printf '%s\n' "${GHCR_IMAGE}:dev" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${GHCR_IMAGE}:latest"
+          fi
+          printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
 
       - name: Create Docker Hub manifest
         shell: bash
@@ -250,8 +320,12 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
-          docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
-          docker buildx imagetools inspect "${GHCR_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
+          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
+          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
 
       - name: Update README
         uses: peter-evans/dockerhub-description@v4

.github/workflows/homebrew.yml

@@ -1,229 +0,0 @@
-name: Build Homebrew formula
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'brew_dist/**'
-      - 'bin/build_brew.sh'
-      - 'bin/release_brew.sh'
-      - '.github/workflows/homebrew.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: read
-
-jobs:
-  build-and-test:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-latest, ubuntu-24.04]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Validate formula template syntax
-        run: ruby -c brew_dist/archivebox.rb
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies (Linux)
-        if: runner.os == 'Linux'
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local sdist
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --sdist --out-dir /tmp/sdist/
-
-      - name: Generate formula from local sdist
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)"
-          SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')"
-
-          # Install archivebox + poet into a temp venv to generate resource stanzas
-          python3 -m venv /tmp/poet-venv
-          source /tmp/poet-venv/bin/activate
-          pip install --quiet "$SDIST_PATH" homebrew-pypi-poet
-          echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
-          RESOURCES="$(poet archivebox)"
-          deactivate
-
-          # For CI: use file:// URL pointing to local sdist
-          # For release: this gets overridden with the PyPI URL
-          SDIST_URL="file://${SDIST_PATH}"
-
-          cat > /tmp/archivebox.rb << RUBY
-class Archivebox < Formula
-  include Language::Python::Virtualenv
-
-  desc "Self-hosted internet archiving solution"
-  homepage "https://github.com/ArchiveBox/ArchiveBox"
-  url "${SDIST_URL}"
-  sha256 "${SDIST_SHA256}"
-  license "MIT"
-
-  depends_on "python@3.13"
-
-${RESOURCES}
-
-  def install
-    virtualenv_install_with_resources
-  end
-
-  def post_install
-    data_dir = var/"archivebox"
-    data_dir.mkpath
-    ENV["DATA_DIR"] = data_dir.to_s
-    system bin/"archivebox", "init"
-  end
-
-  def caveats
-    <<~EOS
-      ArchiveBox data is stored in:
-        #{var}/archivebox
-
-      To start archiving, run:
-        cd #{var}/archivebox && archivebox add 'https://example.com'
-
-      To start the web UI:
-        cd #{var}/archivebox && archivebox server 0.0.0.0:8000
-    EOS
-  end
-
-  test do
-    assert_match version.to_s, shell_output("#{bin}/archivebox version")
-  end
-end
-RUBY
-
-          echo "[√] Generated formula:"
-          ruby -c /tmp/archivebox.rb
-          cat /tmp/archivebox.rb
-
-      - name: Install Homebrew (Linux only)
-        if: runner.os == 'Linux'
-        run: |
-          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
-          echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV"
-          echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV"
-          echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV"
-          echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH"
-          echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH"
-
-      - name: Install brew dependencies
-        run: brew install python@3.13
-
-      - name: Install archivebox via brew from local formula
-        run: |
-          brew install --build-from-source --verbose /tmp/archivebox.rb
-
-      - name: Verify archivebox version
-        run: archivebox version
-
-      - name: Test archivebox init
-        run: |
-          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
-          archivebox init
-
-      - name: Test archivebox status
-        run: |
-          cd /tmp/archivebox-test
-          archivebox status
-
-  # On release only: generate the real formula with PyPI URL and push to tap
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: build-and-test
-    runs-on: macos-latest
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Wait for PyPI package availability
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..."
-          for i in $(seq 1 30); do
-            if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then
-              echo "[√] archivebox==${VERSION} is available on PyPI"
-              break
-            fi
-            if [ "$i" -eq 30 ]; then
-              echo "[!] Timed out waiting for PyPI. Trying to install anyway..."
-              break
-            fi
-            echo "    Attempt $i/30 - not yet available, waiting 30s..."
-            sleep 30
-          done
-
-      - name: Generate release formula via build_brew.sh
-        run: ./bin/build_brew.sh
-
-      - name: Test formula install
-        run: |
-          brew install --build-from-source brew_dist/archivebox.rb
-          archivebox version
-
-      - name: Upload formula artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox.rb
-          path: brew_dist/archivebox.rb
-
-      - name: Push to homebrew-archivebox tap
-        env:
-          GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap
-
-          cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \
-            cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb
-
-          cd /tmp/tap
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add -A
-          if git diff --cached --quiet; then
-            echo "[i] No changes to formula, skipping push."
-          else
-            git commit -m "Update archivebox to v${VERSION}"
-            git push origin HEAD
-            echo "[√] Formula pushed to homebrew-archivebox tap"
-          fi

.github/workflows/lint.yml

@@ -4,6 +4,10 @@ on:
   workflow_dispatch:
   push:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   UV_NO_SOURCES: "1"
   PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
@@ -35,12 +39,13 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - name: Install dependencies with uv
         run: |
-          uv sync --all-extras --all-groups --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --all-groups -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[sonic,debug]"
 
       - name: Run prek
         run: uv run --no-sync --no-sources prek run --all-files

.github/workflows/pip.yml

@@ -22,12 +22,14 @@ jobs:
       id-token: write
 
     runs-on: ubuntu-24.04
+    environment: pypi
     steps:
       - uses: actions/checkout@v4
 
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - name: Set up Python
@@ -42,6 +44,44 @@ jobs:
           packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
           version: 1.0
 
+      - name: Wait for released ArchiveBox deps on PyPI
+        run: |
+          python - <<'PY'
+          import json
+          import re
+          import sys
+          import time
+          import tomllib
+          import urllib.request
+
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp:
+                      releases = json.load(resp)["releases"]
+                  if version in releases:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
+          PY
+
       - name: UV install archivebox dev + run sub-dependencies
         run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
 

.github/workflows/release-runner.yml

@@ -10,6 +10,10 @@ permissions:
   contents: write
   id-token: write
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   release-state:
     runs-on: ubuntu-latest
@@ -26,6 +30,7 @@ jobs:
 
       - uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - uses: actions/setup-node@v4

.github/workflows/release.yml

@@ -2,8 +2,8 @@ name: Release
 
 # Orchestrates the full release pipeline:
 #   1. Build and publish pip package to PyPI
-#   2. Build .deb packages and Homebrew formula (in parallel, after pip)
-#   3. Build Docker images (in parallel with deb/brew)
+#   2. Build Homebrew formula after pip
+#   3. Build Docker images in parallel
 #
 # Individual workflows also run on push for CI (see their own triggers).
 # Manual fallback for the hosted release pipeline. The normal prerelease loop
@@ -18,18 +18,16 @@ permissions:
   packages: write
   id-token: write
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   pip:
     name: Publish to PyPI
     uses: ./.github/workflows/pip.yml
     secrets: inherit
 
-  debian:
-    name: Build .deb packages
-    needs: pip
-    uses: ./.github/workflows/debian.yml
-    secrets: inherit
-
   homebrew:
     name: Update Homebrew formula
     needs: pip

.github/workflows/test-parallel.yml

@@ -49,8 +49,16 @@ jobs:
 
             # Extract a display name for the test
             name="main/$(basename $test_file .py | sed 's/^test_//')"
+            needs_chromium=false
+            if grep -Eiq '(chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue)' "$test_file"; then
+              needs_chromium=true
+            fi
+            needs_sonic=false
+            if grep -Eiq "shutil\\.which\\([\"']sonic|SEARCH_BACKEND_ENGINE=.*sonic|worker_sonic" "$test_file"; then
+              needs_sonic=true
+            fi
 
-            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
           done
           json_array+="]"
 
@@ -79,8 +87,12 @@ jobs:
             if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
               needs_chromium=true
             fi
+            needs_sonic=false
+            if [ "$plugin_name" = "search_backend_sonic" ]; then
+              needs_sonic=true
+            fi
 
-            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium}"
+            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
           done
           json_array+="]"
 
@@ -129,7 +141,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -151,13 +163,79 @@ jobs:
 
       - name: Install dependencies with uv
         run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
+
+      - name: Install Chrome through ArchiveBox
+        if: ${{ matrix.test.needs_chromium }}
+        env:
+          MIN_CHROMIUM_MAJOR: "149"
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
+          case "$major" in
+            ''|*[!0-9]*) major=0 ;;
+          esac
+          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
+            exit 1
+          fi
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
+          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+
+      - name: Install Sonic through ArchiveBox
+        if: ${{ matrix.test.needs_sonic }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
+          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
 
       - name: Run test - ${{ matrix.test.name }}
         run: |
           mkdir -p tests/out
+          set +e
           uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
+          status=$?
+          set -e
+          if [ "$status" -eq 5 ]; then
+            echo "No tests collected from ${{ matrix.test.path }}; treating as an empty test module."
+            exit 0
+          fi
+          exit "$status"
 
   plugin-tests:
     name: ${{ matrix.plugin.name }}
@@ -199,7 +277,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -221,58 +299,28 @@ jobs:
 
       - name: Install dependencies with uv
         run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
 
-      - name: Select Chromium for extension-capable CDP
+      - name: Install Chrome through ArchiveBox
         if: ${{ matrix.plugin.needs_chromium }}
         env:
           MIN_CHROMIUM_MAJOR: "149"
           LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
         run: |
           set -euo pipefail
-          mkdir -p "$LIB_DIR"
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
           echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
           echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
 
-          candidate="/usr/bin/chromium"
-          version=""
-          major=0
-          if [ -x "$candidate" ]; then
-            version="$("$candidate" --version || true)"
-            major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
-            case "$major" in
-              ''|*[!0-9]*) major=0 ;;
-            esac
-          fi
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
 
-          if [ "$major" -ge "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Using runner Chromium: $candidate ($version)"
-            echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
-            exit 0
-          fi
-
-          echo "Runner Chromium is unavailable or too old: ${version:-not found}"
-          echo "Installing Playwright Chromium into $LIB_DIR..."
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=npm \
-            --lib="$LIB_DIR" \
-            --postinstall-scripts=false \
-            --min-release-age=0 \
-            --no-cache \
-            --overrides='{"npm":{"install_args":["playwright@next"]}}' \
-            playwright
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=playwright \
-            --lib="$LIB_DIR" \
-            --bin-dir="$LIB_DIR/env/bin" \
-            --min-version="$MIN_CHROMIUM_MAJOR" \
-            chromium
-
-          candidate="$LIB_DIR/env/bin/chromium"
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
           if [ ! -x "$candidate" ]; then
-            echo "Playwright Chromium shim was not created at $candidate" >&2
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
             exit 1
           fi
           version="$("$candidate" --version || true)"
@@ -281,12 +329,37 @@ jobs:
             ''|*[!0-9]*) major=0 ;;
           esac
           if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Playwright Chromium is still too old: ${version:-unknown}" >&2
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
             exit 1
           fi
-          echo "Using Playwright Chromium: $candidate ($version)"
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
           echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
 
+      - name: Install Sonic through ArchiveBox
+        if: ${{ matrix.plugin.needs_sonic }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
+          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
+
       - name: Run plugin tests - ${{ matrix.plugin.name }}
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

.github/workflows/test.yml

@@ -1,6 +1,10 @@
 name: Run tests
 on: [push]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   DOCKER_IMAGE: archivebox-ci
   PYTHONIOENCODING: utf-8
@@ -45,7 +49,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v4
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -68,8 +72,8 @@ jobs:
 
       - name: Install dependencies with uv
         run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
 
       ### Run the tests
       - name: Directory listing for debugging

.github/workflows/update-homebrew-tap.yml

@@ -0,0 +1,35 @@
+name: Update Homebrew tap
+
+on:
+  push:
+    branches: [dev]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: update-homebrew-tap-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dispatch:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Dispatch tap update
+        env:
+          HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
+        run: |
+          if [ -z "$HOMEBREW_TAP_TOKEN" ]; then
+            echo "HOMEBREW_TAP_TOKEN is not configured; tap hourly fallback will update the formula."
+            exit 0
+          fi
+
+          curl --fail-with-body \
+            -X POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${HOMEBREW_TAP_TOKEN}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/ArchiveBox/homebrew-archivebox/dispatches \
+            -d '{"event_type":"archivebox-dev-updated"}'

.gitignore

@@ -63,3 +63,6 @@ ArchiveBox.conf
 # vim
 *.sw?
 .vscode
+
+# Local Claude Code task locks (not project state)
+.claude/scheduled_tasks.lock

.pre-commit-config.yaml

@@ -29,12 +29,12 @@ repos:
     hooks:
       - id: ruff-format
         name: ruff-format
-        entry: uv run --active --no-sources ruff format
+        entry: uv run --active --no-sync --no-sources ruff format
         language: system
         types_or: [python, pyi]
       - id: ruff-check
         name: ruff-check
-        entry: uv run --active --no-sources ruff check --fix
+        entry: uv run --active --no-sync --no-sources ruff check --fix
         language: system
         types_or: [python, pyi]
 

AGENTS.md

@@ -0,0 +1,14 @@
+I found 31 unique `no mocking` prompts in recent Codex history, across 21 session transcripts. Only one transcript literally started with `no mocking`; most had it later as testing guidance. Consolidated advice:
+
+- Tests must hit real user-facing code paths: CLI commands, REST/API calls, browser UI, real hooks, real ArchiveBox data dirs, real pytest fixtures, and real subprocess/binary behavior.
+- No mocking, faking, simulating, monkey patching, handwritten fake objects, fake buses, fake hooks, fake binaries, fake handlers, or direct-post shortcuts when the user path is through UI/extension/CLI.
+- No skipped, xfailed, flaky, or “works around platform” tests. Flakiness is treated as a bug, especially on macOS/browser flows.
+- Prefer live integration tests over narrow unit tests when behavior depends on browsers, binaries, ArchiveBox crawls, plugins, LLMs, or server state.
+- Assertions must validate real correctness: returned values, exit codes, DB rows, filesystem contents, field values, uploaded files, rendered output, and side effects. “No error occurred” or “attribute exists” is not enough.
+- Start fixes with failing red tests that reproduce the missing behavior or regression, then implement the minimal fix and confirm the test passes.
+- Use realistic setup patterns “like a user would”: events + bus + handlers, real browser pages/CDP sessions, real URLs or `pytest-httpserver`, real rows, real snapshots, real installs, real local browser/server state.
+- For ArchiveBox/API tests, use existing `conftest.py` fixtures and test harnesses, real test DB rows/data dirs, and user-facing commands/APIs rather than bespoke helpers.
+- For browser/extension tests, trigger behavior through the real extension UI or actual browser session, not direct posting or mocked browser/session objects.
+- For binary/provider tests, use real binaries and real installs; verify constraints and final installed package metadata, not just install success.
+- For coverage quality, keep tests strict, deterministic, grouped consistently, and use a few larger realistic tests when that gives better surface coverage than many tiny fake unit tests.
+- Avoid weakening test coverage, adding compatibility/shim/fallback layers, or guessing from code shape. Trace root causes, verify assumptions with tests/scripts, and let real type/parse errors surface normally.

CLAUDE.md

@@ -1,497 +0,0 @@
-# Claude Code Development Guide for ArchiveBox
-
-## Quick Start
-
-```bash
-# Set up dev environment (always use uv, never pip directly)
-uv sync --dev --all-extras
-
-# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
-```
-
-## Development Environment Setup
-
-### Prerequisites
-- Python 3.11+ (3.13 recommended)
-- uv package manager
-- A non-root user for running tests (e.g., `testuser`)
-
-### Install Dependencies
-```bash
-uv sync --dev --all-extras  # Always use uv, never pip directly
-```
-
-### Activate Virtual Environment
-```bash
-source .venv/bin/activate
-```
-
-### Common Gotchas
-
-#### File Permissions
-New files created by root need permissions fixed for testuser:
-```bash
-chmod 644 archivebox/tests/test_*.py
-```
-
-#### DATA_DIR Environment Variable
-ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
-
-## Code Style Guidelines
-
-### Naming Conventions for Grep-ability
-Use consistent naming for everything to enable easy grep-ability and logical grouping:
-
-**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
-
-**Examples**:
-```python
-# Filesystem migration methods - all start with fs_
-def fs_migration_needed() -> bool: ...
-def fs_migrate() -> None: ...
-def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
-def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
-def _fs_next_version(current: str) -> str: ...
-
-# Logging methods - ALL must start with log_ or _log
-def log_migration_start(snapshot_id: str) -> None: ...
-def _log_error(message: str) -> None: ...
-def log_validation_result(ok: bool, msg: str) -> None: ...
-```
-
-**Rules**:
-- Group related functions with common prefixes
-- Use `_` prefix for internal/private helpers within the same family
-- ALL logging-related methods MUST start with `log_` or `_log`
-- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
-- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
-
-### Minimize Unique Names and Data Structures
-**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
-
-**Example - GOOD**:
-```python
-# Binary has overrides field
-binary = Binary(overrides={'TIMEOUT': '60s'})
-
-# Binary reuses the same field name and structure
-class Binary(models.Model):
-    overrides = models.JSONField(default=dict)  # Same name, same structure
-```
-
-**Example - BAD**:
-```python
-# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
-class Binary(models.Model):
-    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
-```
-
-**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
-
-## Testing
-
-### CRITICAL: Never Run as Root
-ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
-
-```bash
-# Run all migration tests
-sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
-
-# Run specific test file
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
-
-# Run single test
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
-```
-
-### Test File Structure
-```
-archivebox/tests/
-├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
-├── test_migrations_fresh.py      # Fresh install tests
-├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
-├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
-└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
-```
-
-### Test Writing Standards
-
-#### NO MOCKS - Real Tests Only
-Tests must exercise real code paths:
-- Create real SQLite databases with version-specific schemas
-- Seed with realistic test data
-- Run actual `python -m archivebox` commands via subprocess
-- Query SQLite directly to verify results
-
-**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
-
-#### NO SKIPS
-Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
-
-#### Strict Assertions
-- `init` command must return exit code 0 (not `[0, 1]`)
-- Verify ALL data is preserved, not just "at least one"
-- Use exact counts (`==`) not loose bounds (`>=`)
-
-### Example Test Pattern
-```python
-def test_migration_preserves_snapshots(self):
-    """Migration should preserve all snapshots."""
-    result = run_archivebox(self.work_dir, ['init'], timeout=45)
-    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
-
-    ok, msg = verify_snapshot_count(self.db_path, expected_count)
-    self.assertTrue(ok, msg)
-```
-
-### Testing Gotchas
-
-#### Extractors Disabled for Speed
-Tests disable all extractors via environment variables for faster execution:
-```python
-env['SAVE_TITLE'] = 'False'
-env['SAVE_FAVICON'] = 'False'
-# ... etc
-```
-
-#### Timeout Settings
-Use appropriate timeouts for migration tests (45s for init, 60s default).
-
-### Plugin Testing & Code Coverage
-
-**Target: 80-90% coverage** for critical plugins (screenshot, chrome, singlefile, dom)
-
-```bash
-# Run plugin tests with coverage (both Python + JavaScript)
-bash bin/test_plugins.sh screenshot
-
-# View coverage reports
-bash bin/test_plugins.sh --coverage-report
-# Or individual reports:
-coverage report --show-missing --include='archivebox/plugins/*' --omit='*/tests/*'
-```
-
-#### Plugin Test Structure
-
-Tests are **completely isolated** from ArchiveBox - they replicate production directory structure in temp dirs:
-
-```python
-# Correct production paths:
-# Crawl:    DATA_DIR/users/{username}/crawls/YYYYMMDD/example.com/{crawl-id}/{plugin}/
-# Snapshot: DATA_DIR/users/{username}/snapshots/YYYYMMDD/example.com/{snapshot-uuid}/{plugin}/
-
-with tempfile.TemporaryDirectory() as tmpdir:
-    data_dir = Path(tmpdir)
-
-    # Crawl-level plugin (e.g., chrome launcher)
-    crawl_dir = data_dir / 'users' / 'testuser' / 'crawls' / '20240101' / 'example.com' / 'crawl-123'
-    chrome_dir = crawl_dir / 'chrome'
-    chrome_dir.mkdir(parents=True)
-
-    # Snapshot-level plugin (e.g., screenshot)
-    snapshot_dir = data_dir / 'users' / 'testuser' / 'snapshots' / '20240101' / 'example.com' / 'snap-456'
-    screenshot_dir = snapshot_dir / 'screenshot'
-    screenshot_dir.mkdir(parents=True)
-
-    # Run hook in its output directory
-    result = subprocess.run(
-        ['node', str(SCREENSHOT_HOOK), '--url=https://example.com'],
-        cwd=str(screenshot_dir),
-        env={**get_test_env(), 'EXTRA_CONTEXT': '{"snapshot_id":"snap-456"}'},
-        capture_output=True,
-        timeout=120
-    )
-```
-
-#### Coverage Improvement Loop
-
-To improve from ~20% to 80%+:
-
-1. **Run tests**: `bash bin/test_plugins.sh screenshot` → Shows: `19.1% (13/68 ranges)`
-2. **Identify gaps**: Check hook file for untested paths (session connection vs fallback, config branches, error cases)
-3. **Add tests**: Test both execution paths (connect to session + launch own browser), skip conditions, error cases, config variations
-4. **Verify**: Re-run tests → Should show: `85%+ (58+/68 ranges)`
-
-**Critical**: JavaScript hooks have TWO paths that both must be tested (connect to session ~50% + launch browser ~30% + shared ~20%). Testing only one path = max 50% coverage possible!
-
-## Database Migrations
-
-### Generate and Apply Migrations
-```bash
-# Generate migrations (run from archivebox subdirectory)
-cd archivebox
-./manage.py makemigrations
-
-# Apply migrations to test database
-cd data/
-archivebox init
-```
-
-### Schema Versions
-- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
-- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
-- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
-- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
-
-### Testing a Migration Path
-1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
-2. Seed with realistic test data using `seed_0_X_data()`
-3. Run `archivebox init` to trigger migrations
-4. Verify data preservation with `verify_*` functions
-5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
-
-### Squashed Migrations
-When testing 0.8.x (dev branch), you must record ALL replaced migrations:
-```python
-# The squashed migration replaces these - all must be recorded
-('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
-('core', '0024_auto_20240513_1143'),
-# ... all 52 migrations from 0023-0074 ...
-('core', '0023_new_schema'),  # Also record the squashed migration itself
-```
-
-### Migration Strategy
-- Squashed migrations for clean installs
-- Individual migrations recorded for upgrades from dev branch
-- `replaces` attribute in squashed migrations lists what they replace
-
-### Migration Gotchas
-
-#### Circular FK References in Schemas
-SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
-
-## Plugin System Architecture
-
-### Plugin Dependency Rules
-
-Like other plugins, chrome plugins **ARE NOT ALLOWED TO DEPEND ON ARCHIVEBOX OR DJANGO**.
-However, they are allowed to depend on two shared files ONLY:
-- `archivebox/plugins/chrome/chrome_utils.js` ← source of truth API for all basic chrome ops
-- `archivebox/plugins/chrome/tests/chrome_test_utils.py` ← use for your tests, do not implement launching/killing/pid files/cdp/etc. in python, just extend this file as needed.
-
-### Chrome-Dependent Plugins
-
-Many plugins depend on Chrome/Chromium via CDP (Chrome DevTools Protocol). When checking for script name references or debugging Chrome-related issues, check these plugins:
-
-**Main puppeteer-based chrome installer + launcher plugin**:
-- `chrome` - Core Chrome integration (CDP, launch, navigation)
-
-**Metadata extraction using chrome/chrome_utils.js / CDP**:
-- `dns` - DNS resolution info
-- `ssl` - SSL certificate info
-- `headers` - HTTP response headers
-- `redirects` - Capture redirect chains
-- `staticfile` - Direct file downloads (e.g. if the url itself is a .png, .exe, .zip, etc.)
-- `responses` - Capture network responses
-- `consolelog` - Capture console.log output
-- `title` - Extract page title
-- `accessibility` - Extract accessibility tree
-- `seo` - Extract SEO metadata
-
-**Extensions installed using chrome/chrome_utils.js / controlled using CDP**:
-- `ublock` - uBlock Origin ad blocking
-- `istilldontcareaboutcookies` - Cookie banner dismissal
-- `twocaptcha` - 2captcha CAPTCHA solver integration
-
-**Page-alteration plugins to prepare the content for archiving**:
-- `modalcloser` - Modal dialog dismissal
-- `infiniscroll` - Infinite scroll handler
-
-**Main Extractor Outputs**:
-- `dom` - DOM snapshot extraction
-- `pdf` - Generate PDF snapshots
-- `screenshot` - Generate screenshots
-- `singlefile` - SingleFile archival, can be single-file-cli that launches chrome, or singlefile extension running inside chrome
-
-**Crawl URL parsers** (post-process dom.html, singlefile.html, staticfile, responses, headers, etc. for URLs to re-emit as new queued Snapshots during recursive crawling):
-- `parse_dom_outlinks` - Extract outlinks from DOM (special, uses CDP to directly query browser)
-- `parse_html_urls` - Parse URLs from HTML (doesn't use chrome directly, just reads dom.html)
-- `parse_jsonl_urls` - Parse URLs from JSONL (doesn't use chrome directly, just reads dom.html)
-- `parse_netscape_urls` - Parse Netscape bookmark format (doesn't use chrome directly, just reads dom.html)
-
-### Finding Chrome-Dependent Plugins
-
-```bash
-# Find all files containing "chrom" (case-insensitive)
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d: -f1 | sort -u
-
-# Or get just the plugin names
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d/ -f3 | sort -u
-```
-
-**Note**: This list may not be complete. Always run the grep command above when checking for Chrome-related script references or debugging Chrome integration issues.
-
-## Architecture Notes
-
-### Crawl Model (0.9.x)
-- Crawl groups multiple Snapshots from a single `add` command
-- Each `add` creates one Crawl with one or more Snapshots
-- Seed model was removed - crawls now store URLs directly
-
-## Code Coverage
-
-### Overview
-
-Coverage tracking is enabled for passive collection across all contexts:
-- Unit tests (pytest)
-- Integration tests
-- Dev server (manual testing)
-- CLI usage
-
-Coverage data accumulates in `.coverage` file and can be viewed/analyzed to find dead code.
-
-### Install Coverage Tools
-
-```bash
-uv sync --dev  # Installs pytest-cov and coverage
-```
-
-### Running with Coverage
-
-#### Unit Tests
-```bash
-# Run tests with coverage
-pytest --cov=archivebox --cov-report=term archivebox/tests/
-
-# Or run specific test file
-pytest --cov=archivebox --cov-report=term archivebox/tests/test_migrations_08_to_09.py
-```
-
-#### Dev Server with Coverage
-```bash
-# Start dev server with coverage tracking
-coverage run --parallel-mode -m archivebox server
-
-# Or CLI commands
-coverage run --parallel-mode -m archivebox init
-coverage run --parallel-mode -m archivebox add https://example.com
-```
-
-#### Manual Testing (Always-On)
-To enable coverage during ALL Python executions (passive tracking):
-
-```bash
-# Option 1: Use coverage run wrapper
-coverage run --parallel-mode -m archivebox [command]
-
-# Option 2: Set environment variable (tracks everything)
-export COVERAGE_PROCESS_START=pyproject.toml
-# Now all Python processes will track coverage
-archivebox server
-archivebox add https://example.com
-```
-
-### Viewing Coverage
-
-#### Text Report (Quick View)
-```bash
-# Combine all parallel coverage data
-coverage combine
-
-# View summary
-coverage report
-
-# View detailed report with missing lines
-coverage report --show-missing
-
-# View specific file
-coverage report --include="archivebox/core/models.py" --show-missing
-```
-
-#### JSON Report (LLM-Friendly)
-```bash
-# Generate JSON report
-coverage json
-
-# View the JSON
-cat coverage.json | jq '.files | keys'  # List all files
-
-# Find files with low coverage
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 50) | "\(.key): \(.value.summary.percent_covered)%"'
-
-# Find completely uncovered files (dead code candidates)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# Get missing lines for a specific file
-cat coverage.json | jq '.files["archivebox/core/models.py"].missing_lines'
-```
-
-#### HTML Report (Visual)
-```bash
-# Generate interactive HTML report
-coverage html
-
-# Open in browser
-open htmlcov/index.html
-```
-
-### Isolated Runs
-
-To measure coverage for specific scenarios:
-
-```bash
-# 1. Reset coverage data
-coverage erase
-
-# 2. Run your isolated test/scenario
-pytest --cov=archivebox archivebox/tests/test_migrations_fresh.py
-# OR
-coverage run --parallel-mode -m archivebox add https://example.com
-
-# 3. View results
-coverage combine
-coverage report --show-missing
-
-# 4. Optionally export for analysis
-coverage json
-```
-
-### Finding Dead Code
-
-```bash
-# 1. Run comprehensive tests + manual testing to build coverage
-pytest --cov=archivebox archivebox/tests/
-coverage run --parallel-mode -m archivebox server  # Use the app manually
-coverage combine
-
-# 2. Find files with 0% coverage (strong dead code candidates)
-coverage json
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# 3. Find files with <10% coverage (likely dead code)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 10) | "\(.key): \(.value.summary.percent_covered)%"' | sort -t: -k2 -n
-
-# 4. Generate detailed report for analysis
-coverage report --show-missing > coverage_report.txt
-```
-
-### Tips
-
-- **Parallel mode** (`--parallel-mode`): Allows multiple processes to track coverage simultaneously without conflicts
-- **Combine**: Always run `coverage combine` before viewing reports to merge parallel data
-- **Reset**: Use `coverage erase` to start fresh for isolated measurements
-- **Branch coverage**: Enabled by default - tracks if both branches of if/else are executed
-- **Exclude patterns**: Config in `pyproject.toml` excludes tests, migrations, type stubs
-
-## Debugging Tips
-
-### Check Migration State
-```bash
-sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
-```
-
-### Check Table Schema
-```bash
-sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
-```
-
-### Verbose Test Output
-```bash
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
-```
-
-### Kill Zombie Chrome Processes
-```bash
-./bin/kill_chrome.sh
-```

Dockerfile

@@ -1,38 +1,153 @@
-# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
-#     python3.13, uv, python3-ldap
-#     curl, wget, git, dig, ping, tree, nano
-#     node, npm
-#     ArchiveBox and plugin runtime dependencies installed by archivebox init --install
-# Usage:
-#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
-#     docker build . -t archivebox
-#     docker run -v "$PWD/data":/data archivebox init
-#     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
-#     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
-#     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
-#     docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
-# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+# syntax=docker/dockerfile:1.7
 
+# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
+# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
+# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
+# Build abx-dl first, then point this file at it:
+#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
+#       --build-context abxbus=../abxbus \
+#       --build-context abxpkg=../abxpkg \
+#       --build-context abx-plugins=../abx-plugins \
+#       -t archivebox/abx-dl:dev
+#   docker buildx build . -f Dockerfile \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
+#       -t archivebox:multistage
 
-#########################################################################################
+ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
 
-### Example: Using ArchiveBox in your own project's Dockerfile ########
-
-# FROM python:3.13-slim
-# WORKDIR /data
-# RUN pip install archivebox>=0.9.0   # use latest release here
-# RUN archivebox install
-# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
-
-#########################################################################################
+FROM archivebox/sonic:1.4.9 AS sonic
+FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
 
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
-ARG TARGETVARIANT=
+ARG TARGETVARIANT
 
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ubuntu:24.04
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_COMPILE=1 \
+    PIP_ONLY_BINARY=aiohttp \
+    npm_config_loglevel=error
+
+ENV PYTHON_VERSION=3.13 \
+    NODE_VERSION=24
+
+ENV ARCHIVEBOX_USER=archivebox \
+    DEFAULT_ARCHIVEBOX_UID=911 \
+    DEFAULT_ARCHIVEBOX_GID=911 \
+    IN_DOCKER=True
+
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    CONFIG_DIR=/opt/archivebox \
+    LIB_DIR=/opt/archivebox/lib \
+    ABXPKG_LIB_DIR=/opt/archivebox/lib \
+    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
+    PERSONAS_DIR=/data/personas \
+    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
+    CHROME_HEADLESS=true \
+    CHROME_SANDBOX=false \
+    CHROME_ISOLATION=crawl \
+    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
+
+ENV TMP_DIR=/tmp/archivebox \
+    PIP_VENV_PYTHON=/venv/bin/python3 \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no
+
+ENV HOME=/home/archivebox \
+    XDG_CONFIG_HOME=/home/archivebox/.config \
+    XDG_CACHE_HOME=/home/archivebox/.cache \
+    ABXPKG_INSTALL_TIMEOUT=600 \
+    ABXPKG_POSTINSTALL_SCRIPTS=True \
+    ABXPKG_MIN_RELEASE_AGE=0 \
+    TIMEOUT=600
+
+ENV UV_COMPILE_BYTECODE=false \
+    UV_PYTHON_PREFERENCE=managed \
+    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv \
+    VIRTUAL_ENV=/venv \
+    PATH="/venv/bin:/opt/node/bin:$PATH"
+
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
+WORKDIR "$CODE_DIR"
+
+RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
+    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && uname -a \
+    && sed -n '1,7p' /etc/os-release \
+    && which node && node --version \
+    && which uv && uv self version \
+    ) | tee -a /VERSION.txt
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends ripgrep \
+    && rm -rf /var/lib/apt/lists/*
+
+FROM archivebox-runtime-base AS archivebox-builder
+
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
+    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
+    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
+    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
+    && rm -f /etc/apt/apt.conf.d/docker-clean \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
+    && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \
+    && /usr/bin/uv pip install setuptools pip wheel \
+    && /usr/bin/uv sync \
+        --refresh \
+        --no-dev \
+        --inexact \
+        --no-install-project \
+        --no-install-workspace \
+        --no-sources \
+    && (find /venv/lib/python3.*/site-packages -type f -name '*.so' -exec strip --strip-unneeded {} + 2>/dev/null || true) \
+    && rm -f /venv/bin/uv /venv/bin/uvx \
+    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && COMMIT_HASH="$( \
+        if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
+            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
+            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+                echo "$HEAD_REF"; \
+            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
+                REF_PATH="${HEAD_REF#ref: }"; \
+                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
+            fi; \
+        fi)" \
+    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
+    && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
+    && rm -f /venv/bin/uv /venv/bin/uvx \
+    && (/usr/bin/uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
+    && rm -rf "$CODE_DIR/.git"
+
+FROM archivebox-runtime-base
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -43,347 +158,74 @@ LABEL name="archivebox" \
     org.opencontainers.image.vendor="ArchiveBox" \
     org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
     org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile" \
-    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
-    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
-    com.docker.desktop.extension.api.version=">= 1.4.7" \
-    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
-    com.docker.extension.publisher-url="https://archivebox.io" \
-    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
-    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
-    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
-    com.docker.extension.categories='database,utility-tools'
+    com.docker.image.source.entrypoint="Dockerfile"
 
-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-######### Environment Variables #################################
-
-# Global build-time and runtime environment constants + default pkg manager config
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    npm_config_loglevel=error
-
-# Language Version config
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22.22.3
-
-# Non-root User config
-ENV ARCHIVEBOX_USER="archivebox" \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
-
-# ArchiveBox Source Code + Lib + Data paths
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
-    LIB_BIN_DIR=/opt/archivebox/lib/bin \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/browsers
-
-# Bash SHELL config
-# http://redsymbol.net/articles/unofficial-bash-strict-mode/
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
-
-######### System Environment ####################################
-
-# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
-WORKDIR "$CODE_DIR"
-
-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-# Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && echo \
-    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
-    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
-    && echo \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && which bash && bash --version | sed -n '1p' \
-    && which dpkg && dpkg --version | sed -n '1p' \
-    && echo -e '\n\n' && env && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-# Create non-privileged user for archivebox and chrome
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system $ARCHIVEBOX_USER \
-    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
-    | tee -a /VERSION.txt
-    # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
-    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
-
-# Install system apt dependencies (adding backports to access more recent apt updates)
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
-    && mkdir -p /etc/apt/keyrings \
-    && apt-get update -qq \
-    && apt-get install -qq -y \
-        # 1. packaging dependencies
-        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
-        # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep dnsutils git python3.12-venv \
-        # 3. frivolous CLI helpers to make debugging failed archiving easier
-        tree nano iputils-ping \
-        # nano iputils-ping dnsutils htop procps jq yq
-    && rm -rf /var/lib/apt/lists/*
-
-# Install sonic search backend
 COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-RUN (which sonic && sonic --version) | tee -a /VERSION.txt
 
-######### Language Environments ####################################
+COPY --from=archivebox-builder /opt/uv/python /opt/uv/python
+COPY --from=archivebox-builder /venv /venv
+COPY --from=archivebox-builder /app /app
+COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
 
-# Set up Python environment
-# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
-#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
-    # && apt-get update -qq \
-    # && apt-get install -qq -y --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
-    # && rm -rf /var/lib/apt/lists/* \
-    # tell PDM to allow using global system python site packages
-    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
-    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
-    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
-    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
-    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
-    # && python3 -m venv $GLOBAL_VENV \
-    # install global dependencies / python build dependencies in GLOBAL_VENV
-    # && pip install --upgrade pip setuptools wheel \
-    # Save version info
-    # && ( \
-    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
-    #     && which pip && pip --version \
-    #     # && which pdm && pdm --version \
-    #     && echo -e '\n\n' \
-    # ) | tee -a /VERSION.txt
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
+    && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
+    && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
+    && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
+    && ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
+    && ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
+    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
+    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
+    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
+    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
+    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
+    && (which sonic && sonic --version) | tee -a /VERSION.txt \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.cache" \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER/.cache/abxbus/semaphores" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \
+    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/abxbus/semaphores" \
+    && openssl rand -hex 16 > /etc/machine-id \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
+    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
 
-
-# Set up Node environment from the official platform tarball. This avoids
-# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into
-# emulated Docker builds.
-RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    case "$TARGETARCH" in \
-        amd64) NODE_DIST_ARCH="x64" ;; \
-        arm64) NODE_DIST_ARCH="arm64" ;; \
-        *) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \
-    esac \
-    && NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \
-    && echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \
-    && grep "  ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \
-    && tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \
-    && rm "$NODE_TARBALL" SHASUMS256.txt \
-    # Save version info
-    && ( \
-        which node && node --version \
-        && which npm && npm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-# Set up uv and main app /venv
-RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
-ENV UV_COMPILE_BYTECODE=0 \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv
-WORKDIR "$CODE_DIR"
-# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
-    && uv venv /venv --python ${PYTHON_VERSION}
-ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
-RUN uv pip install setuptools pip \
-    && ( \
-        which python3 && python3 --version \
-        && which uv && uv self version \
-        && uv python find --system && uv python find \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-######### ArchiveBox & Extractor Dependencies ##################################
-
-# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc \
-        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
-        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-    && uv pip install \
-        "python-ldap>=3.4.3" \
-    && apt-get purge -y \
-        python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
-
-
-# Runtime config used by plugin hooks. Plugin binaries and npm packages are
-# installed into LIB_DIR below by archivebox init --install and resolved from
-# LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH.
-ENV PERSONAS_DIR=/data/personas \
-    CHROME_EXTENSIONS_DIR=/opt/archivebox/lib/chrome_extensions \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-######### Build Dependencies ####################################
-
-
-# Install ArchiveBox Python venv dependencies from pyproject.toml.
-RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends build-essential gcc python3-dev \
-    && uv --no-cache sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /var/lib/apt/lists/*
-    # installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies
-
-# Setup ArchiveBox runtime config
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/usr/bin/python3.12 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-WORKDIR "$DATA_DIR"
-RUN openssl rand -hex 16 > /etc/machine-id \
-    && mkdir -p "$DATA_DIR" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" \
-    && mkdir -p "$TMP_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
-    && mkdir -p "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-# Pre-bake plugin-managed runtime dependencies using the same abx-dl installer
-# path users run later, before copying ArchiveBox source so source-only edits do
-# not invalidate the heavy browser/plugin dependency layer.
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/opt/archivebox/lib,sharing=locked,id=archivebox-lib-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
-    && export PERSONAS_DIR="$LIB_DIR/personas" \
-    && export CHROME_EXTENSIONS_DIR="$LIB_DIR/chrome_extensions" \
-    && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
-    && mkdir -p "$LIB_DIR" "$LIB_DIR/chrome_extensions" \
-    && apt-get update -qq \
-    && if [ "$TARGETARCH" = "arm64" ]; then \
-        abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \
-        abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
-    fi \
-    && TIMEOUT=600 PUID=0 PGID=0 abx-dl plugins --install \
-    && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \
-    && mkdir -p /opt/archivebox/lib-layer \
-    && cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \
-    && rm -rf /var/lib/apt/lists/* \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer
-
-RUN rm -rf "$LIB_DIR" \
-    && mv /opt/archivebox/lib-layer "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"
-
-# Install ArchiveBox Python package from the checked-out source.
-WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && pip install \
-        --no-deps \
-        "$CODE_DIR" \
-    && ( \
-        pip show archivebox \
-        && which archivebox \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
-    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
-
-# Initialize an empty image collection without rerunning dependency installs.
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
-    && PUID=0 PGID=0 archivebox init \
-    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
+    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
+    && archivebox init \
+    && (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
         "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
         "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
         "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
         "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        2>/dev/null || true)
+        "$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
+        "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" "/home/$ARCHIVEBOX_USER/.cache" \
+        2>/dev/null || true) \
+    && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
 
-# Print version for nice docker finish summary
-RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+RUN chmod +x "$CODE_DIR"/bin/*.sh \
+    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+
+RUN "$LIB_DIR/playwright/bin/chromium" --version | tee -a /VERSION.txt \
+    && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
+    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
+    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
+    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
+    && for forbidden_bin in gcc g++ make; do ! command -v "$forbidden_bin" || (echo "Unexpected build tool in runtime: $forbidden_bin=$(command -v "$forbidden_bin")" >&2 && exit 1); done \
+    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
+    && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER/.cache" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "/home/$ARCHIVEBOX_USER/.cache/abxbus/semaphores" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "/home/$ARCHIVEBOX_USER/.cache/uv" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install \
+    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+
+RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
     && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
     ) | tee -a /VERSION.txt
 
-# Verify ArchiveBox is installed and write full version/dependency info.
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
-    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install 2>&1 | tee -a /VERSION.txt \
-    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-####################################################
-
-# Expose ArchiveBox's main interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Nick Sweeting
+Copyright (c) 2026 Nick Sweeting
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -295,18 +295,18 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
 <ol>
-<li>Download and install the <code>.deb</code> package from the <a href="https://github.com/ArchiveBox/ArchiveBox/releases">latest release</a>.
-<pre lang="bash"><code style="white-space: pre-line"># download the .deb for your architecture (amd64 or arm64)
-ARCH="$(dpkg --print-architecture)"
-VERSION="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'].lstrip('v'))")"
-curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${VERSION}_${ARCH}.deb" -o /tmp/archivebox.deb
-sudo apt install /tmp/archivebox.deb
-archivebox version                         # make sure all dependencies are installed
+<li>Add the third-party ArchiveBox apt repo and install <code>archivebox</code>.
+<pre lang="bash"><code style="white-space: pre-line">echo 'deb [trusted=yes] https://archivebox.github.io/debian-archivebox dev main' | sudo tee /etc/apt/sources.list.d/archivebox.list
+sudo apt update
+sudo apt install archivebox
+archivebox version                         # make sure the package is installed
 </code></pre>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --install
+archivebox init
+sudo archivebox install
+archivebox add 'https://example.com'
 </code></pre>
 <br/>
 </li>
@@ -476,6 +476,22 @@ For more discussion on managed and paid hosting options see here: <a href="https
 ArchiveBox commands can be run in a terminal [directly on your host](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), or via [Docker](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1)/[Docker Compose](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage).  
 <sup>(depending on how you chose to install it above)</sup>
 
+<!--
+```bash
+set -euo pipefail
+__archivebox_docs_home="$(mktemp -d)"
+export HOME="$__archivebox_docs_home"
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init
+archivebox version
+archivebox help
+test -f index.sqlite3
+test -d archive
+rm -rf "$__archivebox_docs_home"
+```
+-->
+<!--pytest.mark.skip(reason='Mixed host/Docker alternatives include interactive Docker commands')-->
 ```bash
 mkdir -p ~/archivebox/data   # create a new data dir anywhere
 cd ~/archivebox/data         # IMPORTANT: cd into the directory
@@ -512,10 +528,12 @@ archivebox persona create --import=chrome personal
 # make sure you have pip-installed ArchiveBox and it's available in your $PATH first  
 <br/>
 # archivebox [subcommand] [--help]
-archivebox init --install      # safe to run init multiple times (also how you update versions)
+mkdir -p ~/archivebox/data && cd ~/archivebox/data
+archivebox init
+sudo archivebox install
+archivebox add 'https://example.com'
 archivebox version           # get archivebox version info + check dependencies
 archivebox help              # get list of archivebox subcommands that can be run
-archivebox add --depth=1 'https://news.ycombinator.com'
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage">Usage: CLI Usage</a> wiki. ➡️</i>
 </details>
@@ -529,10 +547,11 @@ archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
 # docker compose run archivebox [subcommand] [--help]
-docker compose run archivebox init --install
+docker compose run archivebox init
+docker compose run archivebox install
 docker compose run archivebox version
 docker compose run archivebox help
-docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
+docker compose run archivebox add 'https://example.com'
 # to start webserver: docker compose up
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage">Usage: Docker Compose CLI</a> wiki. ➡️</i>
@@ -547,10 +566,11 @@ docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you create and cd into in a new empty directory first  
 <br/>
 # docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox init --install
+docker run -v $PWD:/data -it archivebox/archivebox init
+docker run -v $PWD:/data -it archivebox/archivebox install
 docker run -v $PWD:/data -it archivebox/archivebox version
 docker run -v $PWD:/data -it archivebox/archivebox help
-docker run -v $PWD:/data -it archivebox/archivebox add --depth=1 'https://news.ycombinator.com'
+docker run -v $PWD:/data -it archivebox/archivebox add 'https://example.com'
 # to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
@@ -676,10 +696,36 @@ docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://exampl
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e1e5bd78-b0b6-45dc-914c-e1046fee4bc4" width="330px" align="right" style="float: right"/>
 
 
+<!--
+```bash
+set -euo pipefail
+__archivebox_docs_home="$(mktemp -d)"
+export HOME="$__archivebox_docs_home"
+mkdir -p ~/archivebox/data ~/Downloads
+cd ~/archivebox/data
+archivebox init
+cat > ~/Downloads/some_feed.xml <<'EOF'
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>ArchiveBox docs test feed</title>
+    <link>https://example.com/</link>
+    <description>ArchiveBox docs test feed</description>
+    <item>
+      <title>Feed item</title>
+      <link>https://example.com/from-feed</link>
+      <guid>https://example.com/from-feed</guid>
+    </item>
+  </channel>
+</rss>
+EOF
+```
+-->
+<!--pytest-codeblocks:cont-->
 ```bash
 # archivebox add --help
 archivebox add 'https://example.com/some/page'
-archivebox add --parser=generic_rss < ~/Downloads/some_feed.xml
+archivebox add --depth=1 --plugins=parse_rss_urls "file://$HOME/Downloads/some_feed.xml"
 archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
 echo 'http://example.com' | archivebox add
 echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add
@@ -689,6 +735,19 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
 # if using Docker Compose, add -T when piping stdin / stdout:
 # echo 'https://example.com' | docker compose run -T archivebox add
 ```
+<!--pytest-codeblocks:cont-->
+<!--
+```bash
+archivebox list --json > snapshots.json
+grep -q 'https://example.com/some/page' snapshots.json
+grep -q 'https://example.com/from-feed' snapshots.json
+grep -q 'https://news.ycombinator.com#2020-12-12' snapshots.json
+grep -q 'http://example.com' snapshots.json
+test -d archive
+test "$(find archive -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" -ge 4
+rm -rf "$__archivebox_docs_home"
+```
+-->
 
 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
 
@@ -1085,7 +1144,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
 </li>
 <li><strong>Try to keep the <code>data/index.sqlite3</code> file on local drive (not a network mount)</strong> or SSD for maximum performance, however the <code>data/archive/</code> folder can be on a network mount or slower HDD.</li>
-<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, you may need to set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid"><code>PUID</code> & <code>PGID</code></a> and <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304">disable <code>root_squash</code></a> on your fileshare server.
+<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, make sure the mounted data directory is writable by its intended owner and consider disabling <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304"><code>root_squash</code></a> on your fileshare server.
 </li>
 </ul>
 
@@ -1095,7 +1154,6 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid">Wiki: Configuration (<code>PUID</code> & <code>GUID</code>)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root">Wiki: Security Overview (Do Not Run as Root)</a></li>
 </ul>
 
@@ -1348,6 +1406,7 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc
 
 First make sure you have `uv` installed: https://docs.astral.sh/uv/getting-started/installation/
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 git clone https://github.com/ArchiveBox/monorepo
 cd monorepo
@@ -1369,6 +1428,7 @@ Repos included in monorepo setup:
 
 #### 2. Option A: Install the Python, JS, and system dependencies directly on your machine
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # Install ArchiveBox runtime dependencies
 mkdir -p data && cd data
@@ -1384,6 +1444,7 @@ archivebox server 0.0.0.0:8000
 
 #### 2. Option B: Build the docker container and use that for development instead
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # Optional: develop via docker by mounting the code dir into the container
 # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
@@ -1413,6 +1474,7 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # set up persistent DEBUG=True for all runs
 archivebox config --set DEBUG=True
@@ -1437,6 +1499,7 @@ https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-dj
 
 If you're looking for the latest `dev` Docker image, it's often available pre-built on Docker Hub, simply pull and use `archivebox/archivebox:dev`.
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 docker pull archivebox/archivebox:dev
 docker run archivebox/archivebox:dev version
@@ -1444,9 +1507,10 @@ docker run archivebox/archivebox:dev version
 ```
 
 ##### Build Branch from Source
-  
+
 You can also build and run any branch yourself from source, for example to build & use `dev` locally:
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # docker-compose.yml:
 services:
@@ -1471,6 +1535,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/lint.sh
 ./bin/test.sh
@@ -1484,6 +1549,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # generate the database migrations after changes to models.py
 cd archivebox/
@@ -1548,6 +1614,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/build.sh
 
@@ -1564,6 +1631,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/release.sh
 

archivebox/__init__.py

@@ -79,16 +79,10 @@ def __getattr__(name: str):
         return VERSION
     if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
         from abx_plugins import get_plugins_dir
+        from .config.constants import CONSTANTS
 
         builtin_plugins_dir = Path(get_plugins_dir()).resolve()
-        user_plugins_dir = (
-            Path(
-                os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR")
-                or os.environ.get("USER_PLUGINS_DIR")
-                or os.environ.get("DATA_DIR", os.getcwd()),
-            )
-            / "custom_plugins"
-        )
+        user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
         plugins = {
             "builtin": builtin_plugins_dir,
             "user": user_plugins_dir,

archivebox/__main__.py

@@ -17,4 +17,4 @@ ASCII_LOGO_MINI = r"""
 """
 
 if __name__ == "__main__":
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main(args=sys.argv[1:])

archivebox/api/auth.py

@@ -51,6 +51,27 @@ def auth_using_token(token: str | None, request: HttpRequest | None = None) -> U
     return user
 
 
+def token_from_request(request: HttpRequest) -> str:
+    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
+    auth_header = request.headers.get("Authorization", "")
+    if not token and auth_header.lower().startswith("bearer "):
+        token = auth_header.split(None, 1)[1].strip()
+    return token
+
+
+def authenticated_user_from_request(request: HttpRequest) -> User | None:
+    user = request.user
+    if user.is_authenticated and user.is_active:
+        return user
+
+    token = token_from_request(request)
+    token_user = auth_using_token(token=token, request=request) if token else None
+    if token_user and token_user.is_active:
+        request.user = token_user
+        return token_user
+    return None
+
+
 def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
     """Given a username and password, check if they are valid and return the corresponding user"""
     user: User | None = None

archivebox/api/models.py

@@ -1,7 +1,7 @@
 __package__ = "archivebox.api"
 
 import secrets
-from archivebox.uuid_compat import uuid7
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 
 from django.conf import settings
 from django.db import models
@@ -17,7 +17,7 @@ def generate_secret_token() -> str:
 
 
 class APIToken(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
@@ -41,7 +41,7 @@ class APIToken(models.Model):
 
 
 class OutboundWebhook(WebhookBase):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)

archivebox/api/urls.py

@@ -7,7 +7,7 @@ from django.shortcuts import redirect
 from django.urls import path
 from django.views.generic.base import RedirectView
 
-from archivebox.core.host_utils import build_web_url
+from archivebox.core.routes_util import build_web_url
 from .v1_api import urls as v1_api_urls
 
 

archivebox/api/v1_api.py

@@ -68,14 +68,14 @@ class NinjaAPIWithIOCapture(NinjaAPI):
         # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
 
         # Add Auth Headers to response
-        api_token_attr = getattr(request, "_api_token", None)
+        api_token_attr = request.__dict__.get("_api_token")
         api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
         token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
 
-        response["X-ArchiveBox-Auth-Method"] = str(getattr(request, "_api_auth_method", "None"))
+        response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
         response["X-ArchiveBox-Auth-Expires"] = token_expiry
         response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
-        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if getattr(request.user, "pk", None) else "None"
+        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
         response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
 
         # import ipdb; ipdb.set_trace()

archivebox/api/v1_cli.py

@@ -8,10 +8,11 @@ from enum import Enum
 from django.http import HttpRequest
 
 from ninja import Router, Schema
+from ninja.errors import HttpError
 from pydantic import Field
 
 from archivebox.misc.util import ansi_to_html
-from archivebox.config.common import get_config
+from archivebox.core.models import SnapshotQuerySet
 
 
 # from .auth import API_AUTH_METHODS
@@ -23,6 +24,7 @@ router = Router(tags=["ArchiveBox CLI Sub-Commands"])
 # Schemas
 
 JSONType = list[Any] | dict[str, Any] | bool | int | str | None
+FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]
 
 
 class CLICommandResponseSchema(Schema):
@@ -34,26 +36,11 @@ class CLICommandResponseSchema(Schema):
     stderr: str
 
 
-class FilterTypeChoices(str, Enum):
-    exact = "exact"
-    substring = "substring"
-    regex = "regex"
-    domain = "domain"
-    tag = "tag"
-    timestamp = "timestamp"
-
-
-class StatusChoices(str, Enum):
-    indexed = "indexed"
-    archived = "archived"
-    unarchived = "unarchived"
-    present = "present"
-    valid = "valid"
-    invalid = "invalid"
-    duplicate = "duplicate"
-    orphaned = "orphaned"
-    corrupted = "corrupted"
-    unrecognized = "unrecognized"
+FilterTypeChoices = Enum(
+    "FilterTypeChoices",
+    {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
+    type=str,
+)
 
 
 class AddCommandSchema(Schema):
@@ -67,19 +54,33 @@ class AddCommandSchema(Schema):
     snapshot_max_size: int = 0
     parser: str = "auto"
     plugins: str = ""
-    update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
+    only_new: bool | None = None
+    update: bool = False
     overwrite: bool = False
     index_only: bool = False
 
 
-class UpdateCommandSchema(Schema):
-    resume: str | None = None
+class SnapshotFilterCommandSchema(Schema):
     after: float | None = 0
-    before: float | None = 999999999999999
+    before: float | None = None
     filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = ["https://example.com"]
+    filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
+    status: str | None = None
+    url__icontains: str | None = None
+    url__istartswith: str | None = None
+    tag: str | None = None
+    crawl_id: str | None = None
+    limit: int | None = None
+    sort: str | None = None
+    search: str | None = None
+
+
+class UpdateCommandSchema(SnapshotFilterCommandSchema):
+    resume: str | None = None
     batch_size: int = 100
     continuous: bool = False
+    index_only: bool = False
+    migrate_only: bool = False
 
 
 class ScheduleCommandSchema(Schema):
@@ -92,35 +93,40 @@ class ScheduleCommandSchema(Schema):
     every: str | None = None
     tag: str = ""
     depth: int = 0
+    only_new: bool | None = None
+    update: bool = False
     overwrite: bool = False
-    update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
     clear: bool = False
 
 
-class ListCommandSchema(Schema):
-    filter_patterns: list[str] | None = ["https://example.com"]
-    filter_type: str = FilterTypeChoices.substring
-    status: StatusChoices = StatusChoices.indexed
-    after: float | None = 0
-    before: float | None = 999999999999999
-    sort: str = "bookmarked_at"
+class ListCommandSchema(SnapshotFilterCommandSchema):
     as_json: bool = True
     as_html: bool = False
     as_csv: str | None = "timestamp,url"
     with_headers: bool = False
 
 
-class RemoveCommandSchema(Schema):
-    after: float | None = 0
-    before: float | None = 999999999999999
+class RemoveCommandSchema(SnapshotFilterCommandSchema):
     filter_type: str = FilterTypeChoices.exact
-    filter_patterns: list[str] | None = ["https://example.com"]
+    timeout: float = 60.0
+
+
+def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
+    kwargs = args.dict()
+    kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
+    kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
+    return kwargs
 
 
 @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
 def cli_add(request: HttpRequest, args: AddCommandSchema):
     from archivebox.cli.archivebox_add import add
 
+    config_overrides: dict[str, object] = {}
+    if args.only_new is not None:
+        config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
     crawl, snapshots = add(
         urls=args.urls,
         snapshot_ids=args.snapshot_ids,
@@ -130,13 +136,12 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         crawl_max_size=args.crawl_max_size,
         crawl_timeout=args.crawl_timeout,
         snapshot_max_size=args.snapshot_max_size,
-        update=args.update,
         index_only=args.index_only,
-        overwrite=args.overwrite,
         plugins=args.plugins,
         parser=args.parser,
         bg=True,  # Always run in background for API calls
         created_by_id=request.user.pk,
+        config=config_overrides or None,
     )
 
     snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
@@ -146,8 +151,8 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         "snapshot_ids": snapshot_ids,
         "queued_urls": args.urls,
     }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
 
     return {
         "success": True,
@@ -161,24 +166,39 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
 
 @router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
 def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import update
+    from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
+    from archivebox.core.snapshot_status import normalize_snapshot_status
 
-    result = update(
-        filter_patterns=args.filter_patterns or [],
-        filter_type=args.filter_type or FilterTypeChoices.substring,
-        after=args.after,
-        before=args.before,
-        resume=args.resume,
-        batch_size=args.batch_size,
-        continuous=args.continuous,
-        stop_daemon_stack=False,
+    try:
+        status = normalize_snapshot_status(args.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+
+    update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    update_kwargs["status"] = status
+    update_kwargs["stop_daemon_stack"] = False
+
+    is_filtered_update = any(
+        (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
     )
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    matched_snapshot_ids = []
+    if is_filtered_update:
+        matched_snapshot_ids = [
+            str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
+        ]
+
+    update(**update_kwargs)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
-        "result": result,
+        "result": {
+            "matched_count": len(matched_snapshot_ids),
+            "snapshot_ids": matched_snapshot_ids,
+        }
+        if is_filtered_update
+        else None,
         "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
         "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
     }
@@ -188,6 +208,11 @@ def cli_update(request: HttpRequest, args: UpdateCommandSchema):
 def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
     from archivebox.cli.archivebox_schedule import schedule
 
+    config_overrides: dict[str, object] = {}
+    if args.only_new is not None:
+        config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
     result = schedule(
         import_path=args.import_path,
         add=args.add,
@@ -199,12 +224,11 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
         every=args.every,
         tag=args.tag,
         depth=args.depth,
-        overwrite=args.overwrite,
-        update=args.update,
+        config=config_overrides or None,
     )
 
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
@@ -217,32 +241,38 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
 
 @router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
 def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_search import search
+    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
-    result = search(
-        filter_patterns=args.filter_patterns,
-        filter_type=args.filter_type,
-        status=args.status,
-        after=args.after,
-        before=args.before,
-        sort=args.sort,
-        csv=args.as_csv,
-        json=args.as_json,
-        html=args.as_html,
-        with_headers=args.with_headers,
-    )
+    search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    as_json = search_kwargs.pop("as_json")
+    as_html = search_kwargs.pop("as_html")
+    as_csv = search_kwargs.pop("as_csv")
+    with_headers = search_kwargs.pop("with_headers")
+    try:
+        snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
 
     result_format = "txt"
-    if args.as_json:
+    if as_json:
         result_format = "json"
-        result = json.loads(result)
-    elif args.as_html:
+        result = [
+            json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
+            for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
+        ]
+    elif as_html:
         result_format = "html"
-    elif args.as_csv:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
+    elif as_csv:
         result_format = "csv"
+        cols = [col.strip() for col in as_csv.split(",") if col.strip()]
+        rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
+        result = "\n".join((",".join(cols), *rows) if with_headers else rows)
+    else:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
 
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
@@ -256,37 +286,26 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
 @router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
 def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
     from archivebox.cli.archivebox_remove import remove
-    from archivebox.cli.archivebox_search import get_snapshots
     from archivebox.core.models import Snapshot
 
-    filter_patterns = args.filter_patterns or []
-    snapshots_to_remove = get_snapshots(
-        filter_patterns=filter_patterns,
-        filter_type=args.filter_type,
-        after=args.after,
-        before=args.before,
-    )
-    removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)]
+    remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
+    timeout_arg = remove_kwargs.pop("timeout")
+    timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
+    try:
+        snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
 
-    remove(
+    result = remove(
         yes=True,  # no way to interactively ask for confirmation via API, so we force yes
         snapshots=snapshots_to_remove,
-        before=args.before,
-        after=args.after,
-        filter_type=args.filter_type,
-        filter_patterns=filter_patterns,
+        timeout=timeout,
     )
-
-    result = {
-        "removed_count": len(removed_snapshot_ids),
-        "removed_snapshot_ids": removed_snapshot_ids,
-        "remaining_snapshots": Snapshot.objects.count(),
-    }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
-        "success": True,
-        "errors": [],
+        "success": bool(result["success"]),
+        "errors": [str(result["error"])] if result["error"] else [],
         "result": result,
         "result_format": "json",
         "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",

archivebox/api/v1_core.py

@@ -29,11 +29,11 @@ from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot, ArchiveResult, Tag
 from archivebox.core.permissions import public_snapshots_queryset
-from archivebox.api.auth import auth_using_token
+from archivebox.api.auth import authenticated_user_from_request
 from archivebox.config.common import get_config
-from archivebox.core.host_utils import build_web_url
-from archivebox.misc.util import validate_url_length
-from archivebox.core.tag_utils import (
+from archivebox.core.routes_util import build_web_url
+from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
+from archivebox.core.tag_util import (
     add_snapshot_counts,
     build_tag_cards,
     delete_tag as delete_tag_record,
@@ -49,7 +49,10 @@ from archivebox.core.tag_utils import (
     rename_tag as rename_tag_record,
 )
 from archivebox.crawls.models import Crawl
-from archivebox.api.v1_crawls import CrawlSchema
+from archivebox.api.v1_crawls import CrawlSchema, get_crawl_by_ref
+from archivebox.search.config import get_search_mode, get_search_mode_backend
+from archivebox.search.query import apply_snapshot_search
+from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status
 
 
 router = Router(tags=["Core Models"])
@@ -77,7 +80,7 @@ class CustomPagination(PaginationBase):
     def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
         limit = min(pagination.limit, 500)
         offset = pagination.offset or (pagination.page * limit)
-        total = queryset.count()
+        total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count()
         total_pages = math.ceil(total / limit)
         current_page = math.ceil(offset / (limit + 1))
         items = queryset[offset : offset + limit]
@@ -210,7 +213,10 @@ class ArchiveResultFilterSchema(FilterSchema):
 @paginate(CustomPagination)
 def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
     """List all ArchiveResult entries matching these filters."""
-    return filters.filter(ArchiveResult.objects.all()).distinct()
+    queryset = filters.filter(ArchiveResult.objects.all())
+    if filters.search or filters.snapshot_tag:
+        return queryset.distinct()
+    return queryset
 
 
 def _uuid_ref_query(field_name: str, ref: str) -> Q:
@@ -271,7 +277,7 @@ def _parse_archiveresult_output_json(output_json: str | None) -> dict[str, Any]
 
 
 def _get_archiveresult_upload_data(request: HttpRequest):
-    cached = getattr(request, "_archiveresult_upload_data", None)
+    cached = request.__dict__.get("_archiveresult_upload_data")
     if cached is not None:
         return cached
 
@@ -466,12 +472,7 @@ def _write_archiveresult_files(
             )
 
         guessed_mime = mimetypes.guess_type(relative_output_path)[0]
-        output_mime_type = (
-            (mime_types[0] if mime_types else "")
-            or getattr(uploaded_file, "content_type", None)
-            or guessed_mime
-            or "application/octet-stream"
-        )
+        output_mime_type = (mime_types[0] if mime_types else "") or uploaded_file.content_type or guessed_mime or "application/octet-stream"
         output_files[relative_output_path] = {
             "extension": PurePosixPath(relative_output_path).suffix.lower().lstrip("."),
             "mimetype": output_mime_type,
@@ -499,7 +500,7 @@ def _write_archiveresult_files(
         guessed_mime = mimetypes.guess_type(saved_output_path)[0]
         output_mime_type = (
             (mime_types[index] if index < len(mime_types) else "")
-            or getattr(uploaded_file, "content_type", None)
+            or uploaded_file.content_type
             or guessed_mime
             or "application/octet-stream"
         )
@@ -636,6 +637,9 @@ def patch_archiveresult(
             normalized_status = latest_result.status
         result.status = normalized_status
         update_fields.append("status")
+    elif latest_result.status == ArchiveResult.StatusChoices.QUEUED and ArchiveResult.output_files_upload_complete(output_files):
+        result.status = ArchiveResult.StatusChoices.SUCCEEDED
+        update_fields.append("status")
     if output_json:
         result.output_json = _parse_archiveresult_output_json(output_json)
         update_fields.append("output_json")
@@ -697,7 +701,7 @@ class SnapshotSchema(Schema):
 
     @staticmethod
     def resolve_archiveresults(obj, context):
-        if bool(getattr(context["request"], "with_archiveresults", False)):
+        if bool(context["request"].__dict__.get("with_archiveresults", False)):
             return obj.archiveresult_set.all().distinct()
         return ArchiveResult.objects.none()
 
@@ -783,7 +787,8 @@ def _filter_snapshots_for_rss(
     )
     crawl_id = crawl_id.strip()
     if crawl_id:
-        queryset = queryset.filter(crawl__id__icontains=crawl_id)
+        matching_crawl_pks = list(filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).values_list("pk", flat=True)[:100])
+        queryset = queryset.filter(crawl_id__in=matching_crawl_pks)
 
     created_by = created_by.strip()
     if created_by:
@@ -842,7 +847,7 @@ def _snapshots_rss_response(
 
 
 class SnapshotFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup(["id__icontains", "timestamp__startswith"])] = None
+    id: Annotated[str | None, FilterLookup(["id__istartswith", "id__iendswith", "timestamp__startswith"])] = None
     created_by_id: Annotated[str | None, FilterLookup("crawl__created_by_id")] = None
     created_by_username: Annotated[str | None, FilterLookup("crawl__created_by__username__icontains")] = None
     created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
@@ -851,10 +856,9 @@ class SnapshotFilterSchema(FilterSchema):
     modified_at: Annotated[datetime | None, FilterLookup("modified_at")] = None
     modified_at__gte: Annotated[datetime | None, FilterLookup("modified_at__gte")] = None
     modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
-    search: Annotated[
-        str | None,
-        FilterLookup(["url__icontains", "title__icontains", "tags__name__icontains", "id__icontains", "timestamp__startswith"]),
-    ] = None
+    search: str | None = None
+    search_mode: str | None = None
+    status: str | None = None
     url: Annotated[str | None, FilterLookup("url")] = None
     tag: Annotated[str | None, FilterLookup("tags__name")] = None
     title: Annotated[str | None, FilterLookup("title__icontains")] = None
@@ -862,14 +866,44 @@ class SnapshotFilterSchema(FilterSchema):
     bookmarked_at__gte: Annotated[datetime | None, FilterLookup("bookmarked_at__gte")] = None
     bookmarked_at__lt: Annotated[datetime | None, FilterLookup("bookmarked_at__lt")] = None
 
+    def filter_search(self, value: str | None) -> Q:
+        return Q()
+
+    def filter_search_mode(self, value: str | None) -> Q:
+        return Q()
+
+    def filter_status(self, value: str | None) -> Q:
+        return Q()
+
 
 @router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
 @paginate(CustomPagination)
 def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
     """List all Snapshot entries matching these filters."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.all()
-    return filters.filter(queryset).distinct()
+    try:
+        queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+    queryset = filters.filter(queryset).distinct()
+    query = (filters.search or "").strip()
+    if not query:
+        return queryset
+
+    runtime_config = request.archivebox_config
+    search_mode = get_search_mode(filters.search_mode, config=runtime_config)
+    try:
+        return apply_snapshot_search(
+            queryset,
+            query,
+            search_mode=search_mode,
+            config=runtime_config,
+            include_id_matches=True,
+        )
+    except Exception:
+        if get_search_mode_backend(search_mode, config=runtime_config):
+            return queryset.none()
+        return apply_snapshot_search(queryset, query, search_mode="meta", config=runtime_config, include_id_matches=True)
 
 
 @router.get("/snapshots.rss", url_name="get_snapshots_rss")
@@ -894,18 +928,16 @@ def get_snapshots_rss(
 def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
     """Get a specific Snapshot by id."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.all()
-    try:
-        return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        return queryset.get(_uuid_ref_query("id", snapshot_id))
+    return _get_snapshot_by_ref(snapshot_id)
 
 
 @router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
 def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     tags = normalize_tag_list(data.tags)
-    if data.status is not None and data.status not in Snapshot.StatusChoices.values:
-        raise HttpError(400, f"Invalid status: {data.status}")
+    try:
+        status = normalize_snapshot_status(data.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
     if not data.url.strip():
         raise HttpError(400, "URL is required")
     try:
@@ -916,7 +948,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
         raise HttpError(400, "depth must be between 0 and 4")
 
     if data.crawl_id:
-        crawl = Crawl.objects.get(id__icontains=data.crawl_id)
+        crawl = get_crawl_by_ref(data.crawl_id)
         crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
         tags = tags or crawl_tags
     else:
@@ -933,7 +965,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
         "depth": data.depth,
         "title": data.title,
         "timestamp": str(timezone.now().timestamp()),
-        "status": data.status or Snapshot.StatusChoices.QUEUED,
+        "status": status or Snapshot.StatusChoices.QUEUED,
         "retry_at": timezone.now(),
     }
     snapshot, _ = Snapshot.objects.get_or_create(
@@ -946,10 +978,8 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     if data.title is not None and snapshot.title != data.title:
         snapshot.title = data.title
         update_fields.append("title")
-    if data.status is not None and snapshot.status != data.status:
-        if data.status not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {data.status}")
-        snapshot.status = data.status
+    if status is not None and snapshot.status != status:
+        snapshot.status = status
         update_fields.append("status")
     if update_fields:
         update_fields.append("modified_at")
@@ -970,10 +1000,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
 @router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
 def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
     """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
-    try:
-        snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        snapshot = Snapshot.objects.get(Q(id__icontains=snapshot_id))
+    snapshot = _get_snapshot_by_ref(snapshot_id)
 
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
@@ -996,9 +1023,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
         raise HttpError(400, f"Invalid action: {action}")
 
     if "status" in payload:
-        if payload["status"] not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        snapshot.status = payload["status"]
+        try:
+            snapshot.status = normalize_snapshot_status(payload["status"])
+        except ValueError as err:
+            raise HttpError(400, str(err)) from err
         if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
             snapshot.retry_at = None
         update_fields.append("status")
@@ -1054,7 +1082,7 @@ class TagSchema(Schema):
     def resolve_created_by_username(obj):
         user_model = get_user_model()
         user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
+        username = user.username
         return username if isinstance(username, str) else str(user)
 
     @staticmethod
@@ -1063,7 +1091,7 @@ class TagSchema(Schema):
 
     @staticmethod
     def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
+        if bool(context["request"].__dict__.get("with_snapshots", False)):
             return obj.snapshot_set.all().distinct()
         return Snapshot.objects.none()
 
@@ -1206,7 +1234,7 @@ def _get_snapshot_for_tag_edit(snapshot_ref: str) -> Snapshot:
     is_full_uuid = len(snapshot_ref.replace("-", "")) == 32 and all(char in "0123456789abcdef-" for char in snapshot_ref)
     if is_full_uuid:
         try:
-            return snapshot_qs.get(pk=snapshot_ref)
+            return snapshot_qs.get(pk=snapshot_ref.replace("-", ""))
         except (Snapshot.DoesNotExist, ValueError):
             pass
 
@@ -1249,6 +1277,7 @@ def search_tags(
         "tags": build_tag_cards(
             query=q,
             request=request,
+            preview_limit=0,
             sort=normalized_sort,
             created_by=normalized_created_by,
             year=normalized_year,
@@ -1266,16 +1295,7 @@ def _public_tag_listing_enabled() -> bool:
 
 
 def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
-    user = getattr(request, "user", None)
-    if getattr(user, "is_authenticated", False):
-        return True
-
-    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-    auth_header = request.headers.get("Authorization", "")
-    if not token and auth_header.lower().startswith("bearer "):
-        token = auth_header.split(None, 1)[1].strip()
-
-    if token and auth_using_token(token=token, request=request):
+    if authenticated_user_from_request(request):
         return True
 
     return _public_tag_listing_enabled()
@@ -1287,8 +1307,8 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
     if not _request_has_tag_autocomplete_access(request):
         raise HttpError(401, "Authentication required")
 
-    public_only = not getattr(request.user, "is_authenticated", False) and not getattr(request, "_api_token", None)
-    queryset = get_matching_tags(q, with_snapshot_counts=False)
+    public_only = not request.user.is_authenticated and not request.__dict__.get("_api_token")
+    queryset = get_matching_tags(q)
     public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
     if public_only:
         queryset = queryset.filter(snapshot_set__id__in=public_snapshots.values("id")).distinct()
@@ -1296,7 +1316,7 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
     add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)
 
     return {
-        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": getattr(tag, "num_snapshots", 0)} for tag in tags],
+        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": tag.__dict__.get("num_snapshots", 0)} for tag in tags],
     }
 
 

archivebox/api/v1_crawls.py

@@ -14,9 +14,17 @@ from ninja import Router, Schema
 from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot
+from archivebox.core.permissions import (
+    PERMISSIONS_PUBLIC,
+    PERMISSIONS_UNLISTED,
+    is_admin_user,
+    normalize_permissions,
+)
+from archivebox.config.common import get_config
 from archivebox.crawls.models import Crawl
+from archivebox.misc.util import filter_queryset_by_uuid_substring
 
-from .auth import API_AUTH_METHODS, auth_using_token
+from .auth import API_AUTH_METHODS, authenticated_user_from_request
 
 router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
 
@@ -50,12 +58,20 @@ class CrawlSchema(Schema):
     def resolve_created_by_username(obj):
         user_model = get_user_model()
         user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
+        username = user.username
         return username if isinstance(username, str) else str(user)
 
+    @staticmethod
+    def resolve_config(obj):
+        # Redact credential values so REST responses can never leak the raw
+        # token/secret/api-key that the operator stored in Crawl.config.
+        from archivebox.config.common import redact_sensitive_config
+
+        return redact_sensitive_config(obj.config)
+
     @staticmethod
     def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
+        if bool(context["request"].__dict__.get("with_snapshots", False)):
             return obj.snapshot_set.all().distinct()
         return Snapshot.objects.none()
 
@@ -106,6 +122,7 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
 
     tags = normalize_tag_list(data.tags, data.tags_str)
     config = dict(data.config or {})
+    config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
     crawl = Crawl.objects.create(
         urls="\n".join(urls),
         max_depth=data.max_depth,
@@ -120,12 +137,16 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
     return crawl
 
 
+def get_crawl_by_ref(crawl_id: str):
+    return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+
+
 @router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
 def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
     """Get a specific Crawl by id."""
     setattr(request, "with_snapshots", with_snapshots)
     setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl = get_crawl_by_ref(crawl_id)
 
     if crawl and as_rss:
         query = request.GET.copy()
@@ -137,21 +158,23 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn
 
 
 def crawl_file(request: HttpRequest, crawl_id: str, path: str):
-    user = getattr(request, "user", None)
-    is_superuser = bool(
-        getattr(user, "is_authenticated", False) and getattr(user, "is_active", False) and getattr(user, "is_superuser", False),
-    )
-    if not is_superuser:
-        token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-        auth_header = request.headers.get("Authorization", "")
-        if not token and auth_header.lower().startswith("bearer "):
-            token = auth_header.split(None, 1)[1].strip()
-        token_user = auth_using_token(token=token, request=request) if token else None
-        is_superuser = bool(token_user and token_user.is_active and token_user.is_superuser)
-    if not is_superuser:
-        raise HttpError(403, "Permission denied")
+    # Try to resolve the crawl first; if it doesn't exist, return 404.
+    try:
+        crawl = get_crawl_by_ref(crawl_id)
+    except Crawl.DoesNotExist:
+        raise HttpError(404, "Crawl not found")
+
+    user = authenticated_user_from_request(request)
+
+    # Gate access using the same model as SnapshotView/can_view_snapshot:
+    # admins always pass; owners can see their own crawls; otherwise the crawl
+    # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
+    if not is_admin_user(request):
+        permissions = normalize_permissions(crawl.permissions)
+        is_owner = bool(user and crawl.created_by_id == user.id)
+        if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
+            raise HttpError(404, "Crawl not found")
 
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
     crawl_root = Path(crawl.output_dir).resolve()
     file_path = (crawl_root / path).resolve()
     if not file_path.is_file() or crawl_root not in file_path.parents:
@@ -183,7 +206,7 @@ def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfol
 @router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
 def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
     """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl = get_crawl_by_ref(crawl_id)
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
 
@@ -209,25 +232,23 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
     if "status" in payload:
         if payload["status"] not in Crawl.StatusChoices.values:
             raise HttpError(400, f"Invalid status: {payload['status']}")
+        if payload["status"] == Crawl.StatusChoices.SEALED:
+            crawl.cancel()
+            return crawl
         crawl.status = payload["status"]
-        if crawl.status == Crawl.StatusChoices.SEALED and "retry_at" not in payload:
-            crawl.retry_at = None
         update_fields.append("status")
 
     if "retry_at" in payload:
         crawl.retry_at = payload["retry_at"]
         update_fields.append("retry_at")
 
-    if payload.get("status") == Crawl.StatusChoices.SEALED:
-        crawl.cancel()
-    else:
-        crawl.save(update_fields=update_fields)
+    crawl.save(update_fields=update_fields)
     return crawl
 
 
 @router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
 def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl = get_crawl_by_ref(crawl_id)
     crawl_id_str = str(crawl.id)
     snapshot_count = crawl.snapshot_set.count()
     deleted_count, _ = crawl.delete()

archivebox/api/v1_personas.py

@@ -8,8 +8,10 @@ from uuid import UUID
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.pagination import paginate
 from pydantic import Field
 
+from archivebox.api.v1_core import CustomPagination
 from archivebox.personas.importers import validate_persona_name
 from archivebox.personas.models import Persona
 
@@ -51,6 +53,14 @@ class PersonaSchema(Schema):
     def resolve_created_by_username(obj) -> str:
         return obj.created_by.username
 
+    @staticmethod
+    def resolve_config(obj):
+        # Redact credential values so REST responses don't leak the raw
+        # token/secret/api-key the operator stored in Persona.config.
+        from archivebox.config.common import redact_sensitive_config
+
+        return redact_sensitive_config(obj.config)
+
 
 class PersonaSyncResponseSchema(Schema):
     success: bool
@@ -105,6 +115,7 @@ def find_persona(extension_persona_id: str, name: str) -> Persona | None:
 
 
 @router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
+@paginate(CustomPagination)
 def get_personas(request: HttpRequest):
     """List personas available on this ArchiveBox server."""
     return Persona.objects.all().order_by("name")
@@ -128,7 +139,7 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     created = persona is None
     if persona is None:
         persona = Persona(name=name)
-        if getattr(request.user, "is_authenticated", False):
+        if request.user.is_authenticated:
             persona.created_by = request.user
 
     persona.config = {

archivebox/base_models/admin.py

@@ -3,18 +3,50 @@
 __package__ = "archivebox.base_models"
 
 import json
+import uuid
 from collections.abc import Mapping
-from typing import NotRequired, TypedDict
+from typing import NotRequired, TypedDict, cast
 
 from django import forms
 from django.contrib import admin
 from django.db import models
 from django.forms.renderers import BaseRenderer
 from django.http import HttpRequest, QueryDict
+from django.urls import path, register_converter
 from django.utils.safestring import SafeString, mark_safe
 from django_object_actions import DjangoObjectActions
 
 
+class HexUUIDConverter:
+    """URL path converter that canonicalizes UUIDs to their 32-char hex form.
+
+    Accepts both the hyphenated (``aaaaaaaa-bbbb-...``) and bare-hex
+    (``aaaaaaaabbbb...``) UUID strings on the way in (Django's UUIDField
+    parses either), but ``to_url`` always emits the bare-hex form. This is
+    what makes ``reverse("admin:app_model_change", args=[obj.pk])`` produce
+    ``/admin/app/model/06a1a8facb0d.../change/`` instead of the default
+    hyphenated rendering — admin links throughout the app reverse through
+    this converter once ``BaseModelAdmin.get_urls`` swaps in
+    ``<hexuuid:object_id>`` below.
+    """
+
+    regex = r"[0-9a-fA-F]{32}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
+
+    def to_python(self, value: str) -> str:
+        # Strip hyphens but stay as a string — Django admin treats the
+        # captured object_id as a string and calls ``model._meta.pk.to_python``
+        # itself, so we don't want to short-circuit that.
+        return value.replace("-", "")
+
+    def to_url(self, value) -> str:
+        if isinstance(value, uuid.UUID):
+            return value.hex
+        return str(value).replace("-", "")
+
+
+register_converter(HexUUIDConverter, "hexuuid")
+
+
 class ConfigOption(TypedDict):
     plugin: str
     type: str | list[str]
@@ -44,39 +76,23 @@ class KeyValueWidget(forms.Widget):
     def _get_config_options(self) -> dict[str, ConfigOption]:
         """Get available config options from plugins."""
         try:
-            from archivebox.config.common import ArchiveBoxConfig
-            from archivebox.hooks import discover_plugin_configs
+            from archivebox.config.common import config_field_metadata
 
             options: dict[str, ConfigOption] = {}
-            skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "CRAWL_OUTPUT_DIR", "SNAP_DIR"}
-            for key, field in ArchiveBoxConfig.model_fields.items():
-                if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys:
-                    continue
-                default = field.default
-                try:
-                    json.dumps(default)
-                except TypeError:
-                    default = str(default)
-                options[key] = {
-                    "plugin": "archivebox",
-                    "type": str(field.annotation),
-                    "default": default,
-                    "description": field.description or "",
+            for key, metadata in config_field_metadata().items():
+                option_type = metadata.get("type", "string")
+                option: ConfigOption = {
+                    "plugin": str(metadata.get("plugin", "archivebox")),
+                    "type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
+                    "default": metadata.get("default", ""),
+                    "description": str(metadata.get("description", "")),
                 }
-
-            plugin_configs = discover_plugin_configs()
-            for plugin_name, schema in plugin_configs.items():
-                for key, prop in schema.get("properties", {}).items():
-                    option: ConfigOption = {
-                        "plugin": plugin_name,
-                        "type": prop.get("type", "string"),
-                        "default": prop.get("default", ""),
-                        "description": prop.get("description", ""),
-                    }
+                schema = metadata.get("schema")
+                if isinstance(schema, Mapping):
                     for schema_key in ("enum", "pattern", "minimum", "maximum"):
-                        if schema_key in prop:
-                            option[schema_key] = prop[schema_key]
-                    options[key] = option
+                        if schema_key in schema:
+                            option[schema_key] = schema[schema_key]
+                options[key] = option
             return options
         except Exception:
             return {}
@@ -227,9 +243,6 @@ class KeyValueWidget(forms.Widget):
                             return 'Example: ["value"]';
                         }}
                         if (types.includes('object')) {{
-                            if (key === 'SAVE_ALLOWLIST' || key === 'SAVE_DENYLIST') {{
-                                return 'Example: {{"^https://example\\\\.com": ["wget"]}}';
-                            }}
                             return 'Example: {{"key": "value"}}';
                         }}
                         return '';
@@ -238,8 +251,6 @@ class KeyValueWidget(forms.Widget):
                     function isRegexConfigKey_{widget_id}(key) {{
                         return key === 'URL_ALLOWLIST' ||
                             key === 'URL_DENYLIST' ||
-                            key === 'SAVE_ALLOWLIST' ||
-                            key === 'SAVE_DENYLIST' ||
                             key.endsWith('_PATTERN') ||
                             key.includes('REGEX');
                     }}
@@ -633,17 +644,74 @@ class KeyValueWidget(forms.Widget):
 
                     window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
 
+                    function focusConfigKeyFromHash_{widget_id}() {{
+                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
+                        // to (or creates) the matching row in this editor. Used by
+                        // the in-banner "pin via admin" link and the
+                        // ``→ Edit <KEY> in Machine.config`` shortcut on the live
+                        // config detail page.
+                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
+                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
+                            return;
+                        }}
+                        var container = document.getElementById('{widget_id}_rows');
+                        if (!container) {{
+                            return;
+                        }}
+                        var match = null;
+                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
+                            if (match) {{ return; }}
+                            var keyInput = row.querySelector('.kv-key');
+                            if (keyInput && keyInput.value.trim() === hash) {{
+                                match = row;
+                            }}
+                        }});
+                        if (!match) {{
+                            // No existing row for this key — prepopulate one with the
+                            // key filled in but value left blank so the operator just
+                            // types/pastes the value and hits save.
+                            window.addKeyValueRow_{widget_id}();
+                            var rows = container.querySelectorAll('.key-value-row');
+                            match = rows[rows.length - 1];
+                            var keyInput = match.querySelector('.kv-key');
+                            if (keyInput) {{
+                                keyInput.value = hash;
+                                keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
+                            }}
+                        }}
+                        if (!match) {{
+                            return;
+                        }}
+                        match.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
+                        var prevOutline = match.style.outline;
+                        match.style.outline = '2px solid #f59e0b';
+                        match.style.outlineOffset = '2px';
+                        match.style.transition = 'outline 1.2s ease-out';
+                        setTimeout(function() {{
+                            match.style.outline = prevOutline || 'none';
+                        }}, 1400);
+                        var valueInput = match.querySelector('.kv-value');
+                        if (valueInput) {{
+                            valueInput.focus();
+                            try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}}
+                        }}
+                    }}
+
                     // Initialize on load
                     document.addEventListener('DOMContentLoaded', function() {{
                         initializeRows_{widget_id}();
                         updateHiddenField_{widget_id}();
+                        focusConfigKeyFromHash_{widget_id}();
                     }});
                     // Also run immediately in case DOM is already ready
                     if (document.readyState !== 'loading') {{
                         initializeRows_{widget_id}();
                         updateHiddenField_{widget_id}();
+                        focusConfigKeyFromHash_{widget_id}();
                     }}
 
+                    window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id});
+
                     // Update on any input change
                     var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
 
@@ -670,12 +738,34 @@ class KeyValueWidget(forms.Widget):
         return mark_safe(html)
 
     def _render_row(self, widget_id: str, key: str, value: str) -> str:
+        from archivebox.config.common import is_sensitive_config_key
+
+        # Sensitive keys (``*TOKEN*``, ``*SECRET*``, ``*API_KEY*``, ``*APIKEY*``) are
+        # rendered write-only: the input is a password field with a placeholder
+        # showing the value is set, but the raw value is NEVER sent to the browser.
+        # When the user submits the form with the field left blank, the
+        # ``ConfigEditorMixin.save_model`` hook re-merges the previously-saved
+        # value so leaving it untouched is a no-op rather than a destructive clear.
+        is_sensitive = is_sensitive_config_key(key)
+        has_value = bool(value)
+        if is_sensitive:
+            input_type = "password"
+            rendered_value = ""
+            placeholder = (
+                "•••••• (saved — enter new value to replace, clear by deleting row)" if has_value else "value (will be saved write-only)"
+            )
+            extra_attrs = ' autocomplete="off" data-sensitive="1"' + (' data-had-value="1"' if has_value else "")
+        else:
+            input_type = "text"
+            rendered_value = self._escape(value)
+            placeholder = "value"
+            extra_attrs = ""
         return f'''
             <div class="key-value-row" style="margin-bottom: 6px;">
                 <div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">
                     <input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
                            style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <input type="text" class="kv-value" value="{self._escape(value)}" placeholder="value"
+                    <input type="{input_type}" class="kv-value" value="{rendered_value}" placeholder="{self._escape(placeholder)}"{extra_attrs}
                            style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
                     <datalist class="kv-value-options"></datalist>
                     <button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
@@ -719,6 +809,37 @@ class ConfigEditorMixin(admin.ModelAdmin):
             kwargs["widget"] = KeyValueWidget()
         return super().formfield_for_dbfield(db_field, request, **kwargs)
 
+    def save_model(self, request: HttpRequest, obj, form, change):
+        """Preserve write-only redacted credentials on save.
+
+        The KeyValueWidget renders sensitive keys (``*TOKEN*``, ``*SECRET*``,
+        ``*API_KEY*``, ``*APIKEY*``) with an empty value + password input —
+        the real value never leaves the server. On submit, an empty value
+        for a sensitive key that was previously set means "leave untouched",
+        not "clear it." We honor that here by re-merging the stored value
+        before the row is written. Explicitly removing the row in the UI
+        still clears it (the key is gone from the submitted JSON, so there's
+        nothing to merge over).
+        """
+        from archivebox.config.common import is_sensitive_config_key
+
+        if change and obj.pk and obj.config is not None:
+            try:
+                stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
+            except Exception:
+                stored = {}
+            if isinstance(stored, dict):
+                new_config = dict(obj.config or {})
+                for key, new_value in list(new_config.items()):
+                    if not is_sensitive_config_key(key):
+                        continue
+                    if new_value not in (None, "") and new_value != "********":
+                        continue
+                    if key in stored:
+                        new_config[key] = stored[key]
+                obj.config = new_config
+        super().save_model(request, obj, form, change)
+
 
 class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
     list_display = ("id", "created_at", "created_by")
@@ -741,3 +862,30 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
         if "created_by" in form.base_fields:
             form.base_fields["created_by"].initial = request.user
         return form
+
+    def get_urls(self):
+        """Swap the per-object admin URLs from ``<path:object_id>`` to
+        ``<hexuuid:object_id>`` so canonical change/delete/history URLs use the
+        32-char hex form. The hyphenated form still resolves because the
+        converter's regex accepts both — Django reverses through ``to_url``
+        which always emits hex, so links in templates / changelists / inline
+        formsets all canonicalize automatically.
+
+        Non-UUID PKs (an ``IntegerField`` PK on some legacy table, for example)
+        won't match the converter's regex and fall back to the default
+        ``<path:object_id>`` patterns we still include after our swap.
+        """
+        info = self.opts.app_label, self.opts.model_name
+        object_routes = [
+            path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="%s_%s_history" % info),
+            path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="%s_%s_delete" % info),
+            path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="%s_%s_change" % info),
+        ]
+        # Append after super().get_urls() so our patterns are the
+        # *last-registered* ones with the canonical admin URL names — Django's
+        # reverse() picks the later registration when names collide, which is
+        # how we make ``reverse("admin:app_model_change", args=[obj.pk])``
+        # emit the hex form. The original ``<path:object_id>`` routes stay in
+        # place as a fallback for non-UUID PKs and for resolving inbound
+        # hyphenated URLs (the ``hexuuid`` regex accepts both forms anyway).
+        return super().get_urls() + object_routes

archivebox/base_models/models.py

@@ -2,9 +2,11 @@
 
 __package__ = "archivebox.base_models"
 
+import json
 import shutil
+from typing import Any
 
-from archivebox.uuid_compat import uuid7
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
 
 from django.db import models
@@ -18,6 +20,27 @@ from django.conf import settings
 
 from django_stubs_ext.db.models import TypedModelMeta
 
+from archivebox.config import CONSTANTS
+
+
+def normalize_config_json_values(config: Any) -> Any:
+    if not isinstance(config, dict):
+        return config
+
+    normalized = dict(config)
+    for key, value in list(normalized.items()):
+        if not isinstance(value, str) or len(value) < 2:
+            continue
+        if value[:1] != '"' or value[-1:] != '"':
+            continue
+        try:
+            decoded = json.loads(value)
+        except ValueError:
+            continue
+        if isinstance(decoded, str):
+            normalized[key] = decoded
+    return normalized
+
 
 def get_or_create_system_user_pk(username="system"):
     User = get_user_model()
@@ -44,7 +67,7 @@ class AutoDateTimeField(models.DateTimeField):
 
 
 class ModelWithUUID(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
     created_by = models.ForeignKey(
@@ -92,6 +115,10 @@ class ModelWithHealthStats(models.Model):
     class Meta(TypedModelMeta):
         abstract = True
 
+    @property
+    def admin_change_url(self) -> str:
+        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
+
     @property
     def health(self) -> int:
         total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
@@ -116,6 +143,15 @@ class ModelWithConfig(models.Model):
     class Meta(TypedModelMeta):
         abstract = True
 
+    def save(self, *args, **kwargs):
+        normalized_config = normalize_config_json_values(self.config)
+        if normalized_config != self.config:
+            self.config = normalized_config
+            update_fields = kwargs.get("update_fields")
+            if update_fields is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
+        super().save(*args, **kwargs)
+
 
 class ModelWithDeleteAfter(models.Model):
     delete_after_final_statuses: tuple[str, ...] = ()
@@ -135,7 +171,7 @@ class ModelWithDeleteAfter(models.Model):
     def get_delete_after_config_value(self):
         from archivebox.config.common import get_config
 
-        return get_config(include_machine=False).DELETE_AFTER
+        return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER
 
     def set_delete_at_from_config(self, config_value=None) -> bool:
         if self.delete_at is not None:
@@ -223,9 +259,7 @@ class ModelWithOutputDir(ModelWithUUID):
 
     @classmethod
     def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
-        from archivebox.config.common import get_config
-
-        data_dir = get_config().DATA_DIR.resolve()
+        data_dir = CONSTANTS.DATA_DIR.resolve()
         safe_paths = []
         for raw_path in paths:
             path = Path(raw_path)

archivebox/cli/__init__.py

@@ -16,6 +16,14 @@ if "--debug" in sys.argv:
     os.environ["DEBUG"] = "True"
     sys.argv.remove("--debug")
 
+# Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
+# `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
+# the subcommand executes. Strip it from argv here so each subcommand's own click parser
+# never sees it. Ignored for `help` and `init` themselves.
+if "--init" in sys.argv:
+    sys.argv = [arg for arg in sys.argv if arg != "--init"]
+    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+
 
 class ArchiveBoxGroup(click.Group):
     """lazy loading click group for archivebox commands"""
@@ -58,10 +66,6 @@ class ArchiveBoxGroup(click.Group):
         # Introspection commands
         "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
     }
-    legacy_model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl_compat.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot_compat.main",
-    }
     all_subcommands = {
         **meta_commands,
         **setup_commands,
@@ -73,34 +77,29 @@ class ArchiveBoxGroup(click.Group):
         "import": "add",
         "archive": "add",
     }
-    legacy_model_subcommands = {
-        "crawl": {"create", "list", "update", "delete"},
-        "snapshot": {"create", "list", "update", "delete"},
-    }
 
     @classmethod
     def get_canonical_name(cls, cmd_name):
         return cls.renamed_commands.get(cmd_name, cmd_name)
 
     @classmethod
-    def _should_use_legacy_model_command(cls, cmd_name: str) -> bool:
-        if cmd_name not in cls.legacy_model_commands:
-            return False
+    def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool:
+        wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
+        return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands)
 
-        try:
-            arg_idx = sys.argv.index(cmd_name)
-        except ValueError:
-            return False
+    @classmethod
+    def _setup_django_for_lazy_import(cls, cmd_name: str) -> None:
+        if not cls._needs_django_for_lazy_import(cmd_name):
+            return
 
-        remaining_args = sys.argv[arg_idx + 1 :]
-        if not remaining_args:
-            return False
+        from django.apps import apps
 
-        first_arg = remaining_args[0]
-        if first_arg in ("-h", "--help"):
-            return False
+        if apps.ready:
+            return
 
-        return first_arg not in cls.legacy_model_subcommands[cmd_name]
+        from archivebox.config.django import setup_django
+
+        setup_django()
 
     def get_command(self, ctx, cmd_name):
         # handle renamed commands
@@ -112,11 +111,9 @@ class ArchiveBoxGroup(click.Group):
             cmd_name = new_name
             ctx.invoked_subcommand = cmd_name
 
-        if self._should_use_legacy_model_command(cmd_name):
-            return self._lazy_load(self.legacy_model_commands[cmd_name])
-
         # handle lazy loading of commands
         if cmd_name in self.all_subcommands:
+            self._setup_django_for_lazy_import(cmd_name)
             return self._lazy_load(cmd_name)
 
         # fall-back to using click's default command lookup
@@ -131,9 +128,9 @@ class ArchiveBoxGroup(click.Group):
 
         # print(f'LAZY LOADING {import_path}')
         mod = import_module(modname)
-        func = getattr(mod, funcname)
+        func = vars(mod)[funcname]
 
-        if not hasattr(func, "__doc__"):
+        if func.__doc__ is None:
             raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
 
         # if not isinstance(cmd, click.BaseCommand):
@@ -172,6 +169,16 @@ def cli(ctx, help=False):
             from archivebox.misc.checks import check_data_folder, check_migrations
 
             setup_django()
+            if os.environ.get("ARCHIVEBOX_WANTS_INIT") == "1" and subcommand not in ("init", "help"):
+                # Universal `--init` was passed: build/upgrade the data folder before
+                # the regular preflight runs, so it succeeds on a fresh dir and an
+                # out-of-date schema both. Drop the env var afterwards so spawned
+                # subprocesses (supervisord workers, daphne, runner, etc.) inherit
+                # a clean env and don't re-trigger init in every child.
+                from archivebox.cli.archivebox_init import init as archivebox_init
+
+                archivebox_init(quick=True)
+                os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
             check_data_folder()
             if subcommand != "update":
                 check_migrations(auto_apply=True)
@@ -181,15 +188,12 @@ def cli(ctx, help=False):
                 raise
 
 
-def main(args=None, prog_name=None, stdin=None):
+def main(args=None, prog_name=None):
     # show `docker run archivebox xyz` in help messages if running in docker
     IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
     IS_TTY = sys.stdin.isatty()
     prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")
 
-    # stdin param allows passing input data from caller (used by __main__.py)
-    # currently not used by click-based CLI, but kept for backwards compatibility
-
     previous_unraisablehook = sys.unraisablehook
 
     def ignore_shutdown_unraisable(unraisable):

archivebox/cli/archivebox_add.py

@@ -5,14 +5,14 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox add"
 
 import sys
-import json
 import os
 from pathlib import Path
 
-from typing import TYPE_CHECKING
+from typing import Any, TYPE_CHECKING
 
 import rich_click as click
 
+from archivebox.config import CONSTANTS
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.util import parse_filesize_to_bytes
 
@@ -23,21 +23,28 @@ if TYPE_CHECKING:
     from archivebox.core.models import Snapshot
 
 
-def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
+def _collect_input_urls(args: tuple[str, ...], *, parser: str = "auto") -> list[str]:
     from archivebox.misc.jsonl import read_args_or_stdin
+    from archivebox.misc.util import validate_url
 
     urls: list[str] = []
     for record in read_args_or_stdin(args):
         url = record.get("url")
         if isinstance(url, str) and url:
-            urls.append(url)
+            try:
+                urls.append(validate_url(url))
+            except ValueError as err:
+                raise click.BadParameter(str(err), param_hint="URL") from err
 
         urls_field = record.get("urls")
         if isinstance(urls_field, str):
             for line in urls_field.splitlines():
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    urls.append(line)
+                    try:
+                        urls.append(validate_url(line))
+                    except ValueError as err:
+                        raise click.BadParameter(str(err), param_hint="URL") from err
 
     return urls
 
@@ -58,11 +65,10 @@ def add(
     parser: str = "auto",
     plugins: str = "",
     persona: str = "Default",
-    overwrite: bool = False,
-    update: bool | None = None,
     index_only: bool = False,
     bg: bool = False,
     created_by_id: int | None = None,
+    config: dict[str, Any] | None = None,
 ) -> tuple[Crawl, QuerySet[Snapshot]]:
     """Add a new URL or list of URLs to your archive.
 
@@ -83,14 +89,14 @@ def add(
     crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
     crawl_timeout = int(crawl_timeout or 0)
     snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
-    from archivebox import CONSTANTS
     from archivebox.config.permissions import USER, HOSTNAME
     from archivebox.config.common import get_config
 
-    config = get_config()
+    config_overrides = dict(config or {})
+    runtime_config = get_config()
     crawl_max_concurrent_snapshots_override = crawl_max_concurrent_snapshots is not None
     if crawl_max_concurrent_snapshots is None:
-        crawl_max_concurrent_snapshots = config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
+        crawl_max_concurrent_snapshots = runtime_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
     crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)
 
     if depth not in (0, 1, 2, 3, 4):
@@ -108,6 +114,7 @@ def add(
 
     # import models once django is set up
     from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.personas.models import Persona
     from archivebox.misc.logging_util import printable_filesize
@@ -117,25 +124,8 @@ def add(
 
     created_by_id = created_by_id or get_or_create_system_user_pk()
     started_at = timezone.now()
-    if update is None:
-        update = not config.ONLY_NEW
 
-    if isinstance(urls, str):
-        url_list = [line.strip() for line in urls.splitlines() if line.strip()]
-    else:
-        url_list = [str(url).strip() for url in urls if str(url).strip()]
-    if snapshot_ids and len(snapshot_ids) != len(url_list):
-        raise ValueError("snapshot_ids length must match urls length")
-
-    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
-    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
-    sources_file.parent.mkdir(parents=True, exist_ok=True)
-    if snapshot_ids:
-        sources_file.write_text(
-            "\n".join(json.dumps({"url": url, "id": snapshot_ids[index], "tags": tag, "depth": 0}) for index, url in enumerate(url_list)),
-        )
-    else:
-        sources_file.write_text("\n".join(url_list))
+    source_text = urls if isinstance(urls, str) else "\n".join(str(url) for url in urls)
 
     # 2. Create a new Crawl with inline URLs
     cli_args = [*sys.argv]
@@ -145,8 +135,6 @@ def add(
 
     timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
 
-    # Read URLs directly into crawl
-    urls_content = sources_file.read_text()
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
     persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
@@ -154,9 +142,8 @@ def add(
     effective_persona_config = get_config(persona=persona_obj)
 
     crawl_config = {
-        **({"ONLY_NEW": not update} if not update else {}),
+        "PERMISSIONS": str(effective_persona_config.PERMISSIONS),
         **({"INDEX_ONLY": True} if index_only else {}),
-        **({"OVERWRITE": True} if overwrite else {}),
         **({"PLUGINS": plugins} if plugins else {}),
         **(
             {"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots}
@@ -172,10 +159,21 @@ def add(
         **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
         **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
     }
+    # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
+    # priority for crawl-frozen keys. Runtime-derived execution keys are
+    # stripped by Crawl.save() and rederived when hooks run.
+    crawl_config.update(config_overrides)
 
+    # Crawl.urls is the user's submitted source, not a derived work queue for
+    # this add path. Keeping it byte-for-byte readable is what lets API/UI/CLI
+    # callers audit or resume imports without losing RSS/Netscape/JSON metadata
+    # that is not representable as one plain URL per line.
     crawl = Crawl.objects.create(
-        urls=urls_content,
-        max_depth=depth,
+        urls=source_text,
+        # The internal root snapshot occupies depth 0. URLs discovered from the
+        # submitted source become normal child snapshots at depth 1, so the
+        # effective crawl limit must be one hop deeper than the user requested.
+        max_depth=depth + 1,
         tags_str=tag,
         persona_id=persona_obj.id,
         label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
@@ -185,6 +183,19 @@ def add(
         config=crawl_config,
     )
 
+    # Parser plugins consume this root snapshot through the normal Snapshot
+    # hook lifecycle. ArchiveBox does not select parser plugins or call them
+    # directly; non-parser plugins cheaply no-result unsupported internal input.
+    root_snapshot = Snapshot.objects.create(
+        url=Snapshot.INTERNAL_INPUT_URL,
+        crawl=crawl,
+        depth=0,
+        title="stdin.txt",
+    )
+    staticfile_dir = root_snapshot.output_dir / "staticfile"
+    staticfile_dir.mkdir(parents=True, exist_ok=True)
+    (staticfile_dir / "stdin.txt").write_text(source_text, encoding="utf-8")
+
     print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
     first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
     print(f"    [dim]First URL: {first_url}[/dim]")
@@ -194,8 +205,8 @@ def add(
     #    Discovered URLs become child Snapshots (depth+1)
 
     if index_only:
-        print("[yellow]\\[*] Index-only mode - crawl queued, runner not started[/yellow]")
-        return crawl, crawl.snapshot_set.all()
+        print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
+        return crawl, crawl.snapshot_set.none()
 
     # 5. Start the crawl runner to process the queue
     #    The runner will:
@@ -205,7 +216,7 @@ def add(
     #    - Repeat until max_depth reached
 
     if bg:
-        # Background mode: just queue work and return (background runner via server will pick it up)
+        # Background mode: just queue work and return (background runner via server will pick it up).
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
@@ -213,8 +224,8 @@ def add(
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
         from archivebox.machine.models import Process
-        from archivebox.services.supervision_service import command_owns_runtime_stack, current_command, standby_until_runtime_stack_needed
-        from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
+        from archivebox.core.takeover_util import command_owns_foreground_runner, current_command, standby_until_foreground_runner_needed
+        from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process
 
         command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
         exit_code = 0
@@ -222,15 +233,27 @@ def add(
             try:
                 with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
                     while True:
-                        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
                         exit_code = run_runner_worker(
                             ["--crawl-id", str(crawl.id)],
                             name=f"worker_runner_add_{os.getpid()}",
                             interactive_interrupts=True,
                         )
+                        crawl.refresh_from_db(fields=["status", "retry_at"])
+                        if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
+                            break
+                        # A shared supervisord can be stopped by another
+                        # foreground owner (for example `archivebox server`
+                        # shutting down) and stop the one-shot add runner while
+                        # this add's crawl is still runnable. Keep the add
+                        # process alive so it can re-enter the normal
+                        # foreground-runner ownership loop and finish its crawl.
+                        supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None
+                        if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone):
+                            continue
                         if exit_code == 0:
                             break
-                        if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+                        if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
                             continue
                         raise SystemExit(exit_code)
             except KeyboardInterrupt:
@@ -246,13 +269,14 @@ def add(
         # Print summary for foreground runs
         try:
             crawl.refresh_from_db()
-            snapshots_count = crawl.snapshot_set.count()
             try:
                 from django.db.models import Count, Sum
 
-                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("archiveresult__output_size"))
-                total_bytes = int(totals["total_bytes"] or 0) if totals["snapshot_count"] else 0
+                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("output_size"))
+                snapshots_count = int(totals["snapshot_count"] or 0)
+                total_bytes = int(totals["total_bytes"] or 0)
             except Exception:
+                snapshots_count = crawl.snapshot_set.count()
                 total_bytes, _, _ = get_dir_size(crawl.output_dir)
             total_size = printable_filesize(total_bytes)
             total_time = timezone.now() - started_at
@@ -273,7 +297,7 @@ def add(
             except Exception:
                 rel_output_str = str(crawl.output_dir)
 
-            from archivebox.core.host_utils import build_admin_url
+            from archivebox.core.routes_util import build_admin_url
 
             admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
 
@@ -311,9 +335,16 @@ def add(
 @click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
 @click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
 @click.option("--persona", default="Default", help="Authentication profile to use when archiving")
-@click.option("--overwrite", "-F", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
-@click.option("--update", is_flag=True, default=None, help="Retry any previously skipped/failed URLs when re-adding them")
+@click.option(
+    "--only-new/--no-only-new",
+    "only_new",
+    default=None,
+    help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
+    "Pass --no-only-new to force re-archive of URLs that already exist.",
+)
 @click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
+@click.option("--overwrite", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
+@click.option("--update", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
 @click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
 @click.argument("urls", nargs=-1, type=click.Path())
 @docstring(add.__doc__)
@@ -324,7 +355,12 @@ def main(**kwargs):
 
     with foreground_shutdown_signals(), foreground_parent_watchdog():
         raw_urls = kwargs.pop("urls")
-        urls = _collect_input_urls(raw_urls)
+        if raw_urls:
+            urls = _collect_input_urls(raw_urls, parser=kwargs.get("parser", "auto"))
+        elif not sys.stdin.isatty():
+            urls = sys.stdin.read()
+        else:
+            urls = []
         if not urls:
             raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
         if int(kwargs.get("max_urls") or 0) < 0:
@@ -342,6 +378,16 @@ def main(**kwargs):
         if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
             raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
 
+        # Translate --only-new/--no-only-new into a crawl config override.
+        # add() takes config overrides as a dict; no per-flag kwargs.
+        overwrite = kwargs.pop("overwrite", False)
+        update = kwargs.pop("update", False)
+        only_new = kwargs.pop("only_new", None)
+        if overwrite or update:
+            only_new = False
+        if only_new is not None:
+            kwargs["config"] = {"ONLY_NEW": bool(only_new)}
+
         add(urls=urls, **kwargs)
 
 

archivebox/cli/archivebox_archiveresult.py

@@ -38,7 +38,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict:
@@ -74,7 +74,7 @@ def create_archiveresults(
         1: Failure
     """
     from archivebox.config.common import get_config
-    from archivebox.hooks import discover_hooks
+    from archivebox.plugins.hooks import discover_hooks
     from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
     from archivebox.core.models import Snapshot
 
@@ -152,7 +152,7 @@ def create_archiveresults(
             config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
             hooks = discover_hooks("Snapshot", config=config)
             for hook_path in hooks:
-                hook_name = hook_path.name
+                hook_name = hook_path.stem
                 plugin_name = hook_path.parent.name
                 if not is_tty:
                     write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))

archivebox/cli/archivebox_binary.py

@@ -33,7 +33,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================

archivebox/cli/archivebox_config.py

@@ -6,6 +6,7 @@ import sys
 import toml
 import rich_click as click
 from rich import print
+from pathlib import Path
 
 from archivebox.misc.util import docstring, enforce_types
 from archivebox.misc.toml_util import CustomTOMLEncoder
@@ -30,12 +31,17 @@ def config(
     from archivebox.misc.logging_util import printable_config
     from abx_plugins.plugins.base.utils import resolve_alias
     from archivebox.config.collection import write_config_file
+    from archivebox.config import CONSTANTS_CONFIG
     from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
 
     check_data_folder()
 
     FLAT_CONFIG = get_config().as_dict()
+    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys()
+    readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)}
+    writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config}
+    readable_config = {**writable_config, **readonly_config}
     CONFIGS = get_all_configs()
     plugin_schemas = {
         plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
@@ -56,18 +62,23 @@ def config(
             config_options = [
                 core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
             ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
             for config_section in CONFIGS.values():
-                aliases = getattr(config_section, "aliases", {})
+                aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias}
 
                 for search_key in config_options:
                     # search all aliases in the section
                     for alias_key, key in aliases.items():
-                        if search_key.lower() in alias_key.lower():
+                        if key in readable_config and search_key.lower() in alias_key.lower():
                             matching_config[key] = dict(config_section)[key]
 
                     # search all keys and values in the section
                     for existing_key, value in dict(config_section).items():
+                        if existing_key in readable_config and (
+                            search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower()
+                        ):
+                            matching_config[existing_key] = value
+                    for existing_key, value in readonly_config.items():
                         if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
                             matching_config[existing_key] = value
 
@@ -79,18 +90,18 @@ def config(
             config_options = [
                 core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
             ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
+            failed_config = [key for key in config_options if key not in readable_config]
             if failed_config:
                 print("\n[red][X] These options failed to get[/red]")
                 print("    {}".format("\n    ".join(config_options)))
                 raise SystemExit(1)
         else:
-            matching_config = FLAT_CONFIG
+            matching_config = readable_config
 
         # Display core config sections
         for config_section in CONFIGS.values():
-            section_header = getattr(config_section, "toml_section_header", "")
+            section_header = config_section.toml_section_header
             if isinstance(section_header, str) and section_header:
                 print(f"[grey53]\\[{section_header}][/grey53]")
             else:
@@ -100,12 +111,18 @@ def config(
             print(_format_toml(kv_in_section))
             print("[grey53]################################################################[/grey53]")
 
+        readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config}
+        if readonly_keys:
+            print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
+            print(_format_toml(readonly_keys))
+            print("[grey53]################################################################[/grey53]")
+
         plugin_keys = {}
 
         # Collect all plugin config keys
         for schema in plugin_schemas.values():
             for key in schema.keys():
-                if key in matching_config:
+                if key in matching_config and key in writable_config:
                     plugin_keys[key] = matching_config[key]
 
         # Display all plugin config in single [PLUGINS] section
@@ -135,7 +152,7 @@ def config(
                     f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
                 )
 
-            if key in FLAT_CONFIG:
+            if key in writable_config:
                 new_config[key] = val.strip()
             else:
                 failed_options.append(line)

archivebox/cli/archivebox_crawl.py

@@ -39,7 +39,7 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================
@@ -65,6 +65,7 @@ def create_crawl(
         1: Failure
     """
     from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
+    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
 
@@ -98,7 +99,11 @@ def create_crawl(
         # Collect URLs
         url = record.get("url")
         if url:
-            url_list.append(url)
+            try:
+                url_list.append(validate_url(str(url)))
+            except ValueError as err:
+                rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                return 1
 
         # Handle 'urls' field (newline-separated)
         urls_field = record.get("urls")
@@ -106,7 +111,11 @@ def create_crawl(
             for line in urls_field.split("\n"):
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    url_list.append(line)
+                    try:
+                        url_list.append(validate_url(line))
+                    except ValueError as err:
+                        rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                        return 1
 
     # Output pass-through records first
     if not is_tty:
@@ -263,10 +272,10 @@ def update_crawls(
                 elif status == Crawl.StatusChoices.STARTED:
                     crawl.update_and_requeue(status=Crawl.StatusChoices.STARTED, retry_at=timezone.now())
             if max_depth is not None:
+                crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
                 crawl.max_depth = max_depth
-                crawl.save(update_fields=["max_depth", "modified_at"])
             elif not status:
-                crawl.save(update_fields=["modified_at"])
+                crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
             updated_count += 1
 
             if not is_tty:

archivebox/cli/archivebox_crawl_compat.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_add import add
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--wait/--no-wait", "wait", default=True, help="Accepted for backwards compatibility")
-@click.argument("urls", nargs=-1)
-def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox crawl URL...` entrypoint."""
-    del status, wait
-    add(list(urls), depth=depth, tag=tag, bg=True)
-    sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_extract.py

@@ -48,10 +48,11 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     from rich import print as rprint
     from django.utils import timezone
     from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.services.runner import run_crawl
 
     try:
-        archiveresult = ArchiveResult.objects.get(id=archiveresult_id)
+        archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
     except ArchiveResult.DoesNotExist:
         rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
         return 1
@@ -67,8 +68,14 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
         else:
             # A paused snapshot may still accept explicit maintenance for one
             # ArchiveResult, but this path must not transition it back to
-            # queued/startable work.
-            snapshot.safe_update({"retry_at": timezone.now()}, refresh=False)
+            # queued/startable work. Guard: only set retry_at while the row is
+            # still paused — concurrent resume would otherwise see a stale
+            # retry_at marker.
+            snapshot.safe_update(
+                {"retry_at": timezone.now()},
+                refresh=False,
+                extra_filter={"status": snapshot.StatusChoices.PAUSED},
+            )
         crawl = snapshot.crawl
         if not crawl.claim_processing_lock(lock_seconds=10):
             rprint(
@@ -110,6 +117,7 @@ def run_plugins(
     wait: bool = True,
     emit_results: bool = True,
     show_progress: bool = True,
+    preserve_queued: bool = False,
 ) -> int:
     """
     Run plugins on Snapshots from input.
@@ -203,7 +211,7 @@ def run_plugins(
             if snapshot_id in existing_snapshot_ids
             for plugin_name in plugin_names
         )
-    plugins_by_name = discover_plugins()
+    plugins_by_name = discover_plugins(runtime="archivebox")
     requested_rows: set[tuple[str, str, str]] = set()
     for snapshot_id, plugin_name in requested_pairs:
         plugin = plugins_by_name.get(plugin_name)
@@ -213,6 +221,18 @@ def run_plugins(
         else:
             requested_rows.add((snapshot_id, plugin_name, ""))
 
+    queued_rows: set[tuple[str, str, str]] = set()
+    if preserve_queued and requested_rows:
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+    rows_to_queue = requested_rows - queued_rows
+
     reset_fields = {
         "status": ArchiveResult.StatusChoices.QUEUED,
         "output_str": "",
@@ -224,23 +244,34 @@ def run_plugins(
         "end_ts": None,
         "modified_at": timezone.now(),
     }
-    if plugins_list:
-        ArchiveResult.objects.filter(snapshot_id__in=existing_snapshot_ids, plugin__in=plugins_list).update(**reset_fields)
-    elif requested_plugins_by_snapshot:
-        snapshot_ids_by_plugin: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, plugin_names in requested_plugins_by_snapshot.items():
-            if snapshot_id in existing_snapshot_ids:
-                for plugin_name in plugin_names:
-                    snapshot_ids_by_plugin[plugin_name].add(snapshot_id)
-        for plugin_name, plugin_snapshot_ids in snapshot_ids_by_plugin.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(**reset_fields)
-    existing_rows = set(
-        ArchiveResult.objects.filter(
-            snapshot_id__in=existing_snapshot_ids,
-            plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
-        ).values_list("snapshot_id", "plugin", "hook_name"),
+    if rows_to_queue and plugins_list:
+        rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    elif rows_to_queue and requested_plugins_by_snapshot:
+        snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    existing_rows = (
+        {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        if rows_to_queue
+        else set()
     )
-    missing_rows = requested_rows - {(str(snapshot_id), plugin_name, hook_name) for snapshot_id, plugin_name, hook_name in existing_rows}
+    missing_rows = rows_to_queue - existing_rows
     if missing_rows:
         ArchiveResult.objects.bulk_create(
             [
@@ -265,8 +296,30 @@ def run_plugins(
             # also keep status=paused here: `retry_at` only asks the orchestrator
             # to process the queued plugin rows, and run_due_snapshot restores
             # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
-            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "modified_at"):
-                snapshot.safe_update({"retry_at": queue_at, "modified_at": queue_at}, refresh=False)
+            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
+            if preserve_queued and queued_rows:
+                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
+                    .filter(retry_at__gt=queue_at)
+                    .values_list("id", flat=True)
+                )
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list(
+                        "id",
+                        flat=True,
+                    )
+                )
+            for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
+                # Guard the read-time status so we never bump retry_at on a
+                # row that's been re-queued / started by a concurrent runner.
+                snapshot.safe_update(
+                    {"retry_at": queue_at, "modified_at": queue_at},
+                    refresh=False,
+                    extra_filter={"status": snapshot.status},
+                )
         else:
             # No plugin rows were requested, so this is a full snapshot retry.
             for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):
@@ -278,6 +331,7 @@ def run_plugins(
                         "modified_at": queue_at,
                     },
                     refresh=False,
+                    extra_filter={"status": snapshot.status},
                 )
     if existing_crawl_ids and not requested_rows:
         from archivebox.crawls.models import Crawl
@@ -289,7 +343,11 @@ def run_plugins(
             }
             if crawl.status != Crawl.StatusChoices.STARTED:
                 update_fields["status"] = Crawl.StatusChoices.QUEUED
-            crawl.safe_update(update_fields, refresh=False)
+            crawl.safe_update(
+                update_fields,
+                refresh=False,
+                extra_filter={"status": crawl.status},
+            )
 
     if processed_count == 0:
         rprint("[red]No snapshots to process[/red]", file=sys.stderr)
@@ -361,16 +419,11 @@ def run_plugins(
 
 
 def is_archiveresult_id(value: str) -> bool:
-    """Check if value looks like an ArchiveResult UUID."""
-    import re
-
-    uuid_pattern = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
-    if not uuid_pattern.match(value):
-        return False
-    # Verify it's actually an ArchiveResult (not a Snapshot or other object)
+    """Check if value resolves to an ArchiveResult ID."""
     from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
 
-    return ArchiveResult.objects.filter(id=value).exists()
+    return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists()
 
 
 @click.command()

archivebox/cli/archivebox_help.py

@@ -13,6 +13,10 @@ from rich.panel import Panel
 
 
 def _command_doc(cmd: str, import_path: str) -> str:
+    def first_doc_line(docstring: str | None) -> str:
+        lines = (docstring or "").splitlines()
+        return lines[0] if lines else ""
+
     modname, _ = import_path.rsplit(".", 1)
     spec = importlib.util.find_spec(modname)
     if spec is None or spec.origin is None:
@@ -26,9 +30,9 @@ def _command_doc(cmd: str, import_path: str) -> str:
     for name in (cmd, "main"):
         for node in tree.body:
             if isinstance(node, ast.FunctionDef) and node.name == name:
-                return (ast.get_docstring(node) or "").splitlines()[0]
+                return first_doc_line(ast.get_docstring(node))
 
-    return (ast.get_docstring(tree) or "").splitlines()[0]
+    return first_doc_line(ast.get_docstring(tree))
 
 
 def help() -> None:
@@ -85,8 +89,8 @@ def help() -> None:
     [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
 """)
 
-    config = get_config()
-    if os.access(config.ARCHIVE_DIR, os.R_OK) and config.ARCHIVE_DIR.is_dir():
+    get_config()
+    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
         pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
         EXAMPLE_USAGE = f"""
 [light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]

archivebox/cli/archivebox_init.py

@@ -25,7 +25,7 @@ def _display_data_path(path: Path, data_dir: Path) -> str:
 def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
     """Initialize a new ArchiveBox collection in the current directory"""
 
-    from archivebox.config import CONSTANTS, VERSION, DATA_DIR
+    from archivebox.config import CONSTANTS, VERSION
     from archivebox.config.common import get_config
     from archivebox.config.collection import write_config_file
     from archivebox.misc.db import apply_migrations
@@ -37,7 +37,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
     #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
 
-    is_empty = not len(set(os.listdir(DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
+    is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
     existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
     if is_empty and not existing_index:
         print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
@@ -65,21 +65,23 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     else:
         print("\n[green][+] Building archive folder structure...[/green]")
 
-    archive_path = _display_data_path(config.ARCHIVE_DIR, DATA_DIR)
-    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, DATA_DIR)
-    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, DATA_DIR)
+    archive_path = _display_data_path(CONSTANTS.ARCHIVE_DIR, CONSTANTS.DATA_DIR)
+    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, CONSTANTS.DATA_DIR)
+    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, CONSTANTS.DATA_DIR)
     print(f"    + {archive_path}, {sources_path}, {logs_path}...")
     Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
-    config.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
-    config.USERS_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.USERS_DIR.mkdir(parents=True, exist_ok=True)
     Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
+    for path in (Path(CONSTANTS.SOURCES_DIR), CONSTANTS.ARCHIVE_DIR, CONSTANTS.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
+        path.chmod(int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)
 
-    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, DATA_DIR)}...")
+    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, CONSTANTS.DATA_DIR)}...")
 
     # create the .archivebox_id file with a unique ID for this collection
     from archivebox.config.paths import _get_collection_id
 
-    _get_collection_id(DATA_DIR, force_create=True)
+    _get_collection_id(CONSTANTS.DATA_DIR, force_create=True)
 
     # create the ArchiveBox.conf file
     write_config_file({"SECRET_KEY": config.SECRET_KEY})
@@ -92,14 +94,22 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     from archivebox.config.django import setup_django
 
     setup_django()
-    check_migrations(blocking=True, auto_apply=False)
+    previous_wants_init = os.environ.get("ARCHIVEBOX_WANTS_INIT")
+    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+    try:
+        check_migrations(blocking=True, auto_apply=False)
 
-    for migration_line in apply_migrations(DATA_DIR):
-        sys.stdout.write(f"    {migration_line}\n")
+        for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
+            sys.stdout.write(f"    {migration_line}\n")
+    finally:
+        if previous_wants_init is None:
+            os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
+        else:
+            os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
 
     assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
     print()
-    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, DATA_DIR)}")
+    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
 
     # from django.contrib.auth.models import User
     #     call_command("createsuperuser", interactive=True)
@@ -153,7 +163,6 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
     if working_lib_dir:
         working_lib_dir.mkdir(parents=True, exist_ok=True)
-        (working_lib_dir / "bin").mkdir(parents=True, exist_ok=True)
 
     if install:
         from archivebox.cli.archivebox_install import install as install_method

archivebox/cli/archivebox_install.py

@@ -22,12 +22,16 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
     """
 
     from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
-    from archivebox.config.common import get_config
+    from archivebox.config import CONSTANTS
     from archivebox.misc.logging import stderr
     from archivebox.cli.archivebox_init import init
 
-    config = get_config()
-    archive_dir = config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
+
+    if dry_run:
+        print("[dim]Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow[/dim]")
+        return
+
     if not (os.access(archive_dir, os.R_OK) and archive_dir.is_dir()):
         init()  # must init full index because we need a db to store Binary entries in
 
@@ -47,10 +51,6 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
         print(f"    DATA_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].")
         print()
 
-    if dry_run:
-        print("[dim]Dry run - would run the abx-dl install flow[/dim]")
-        return
-
     # Set up Django
     from archivebox.config.django import setup_django
 

archivebox/cli/archivebox_list.py

@@ -7,50 +7,15 @@ import sys
 
 import rich_click as click
 
-from archivebox.cli.archivebox_snapshot import list_snapshots
+from archivebox.cli.archivebox_snapshot import list_snapshots, snapshot_filter_options, snapshot_output_options
 
 
 @click.command()
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def main(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-) -> None:
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def main(**kwargs) -> None:
     """List Snapshots."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_machine.py

@@ -27,7 +27,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================

archivebox/cli/archivebox_persona.py

@@ -37,7 +37,7 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 from archivebox.personas import importers as persona_importers
 
 
@@ -372,7 +372,7 @@ def create_personas(
         if not is_tty:
             write_record(
                 {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "id": str(persona.id),
                     "name": persona.name,
                     "path": str(persona.path),
                     "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -424,7 +424,7 @@ def list_personas(
         else:
             write_record(
                 {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "id": str(persona.id),
                     "name": persona.name,
                     "path": str(persona.path),
                     "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -500,7 +500,7 @@ def update_personas(name: str | None = None) -> int:
             if not is_tty:
                 write_record(
                     {
-                        "id": str(persona.id) if hasattr(persona, "id") else None,
+                        "id": str(persona.id),
                         "name": persona.name,
                         "path": str(persona.path),
                     },

archivebox/cli/archivebox_pluginmap.py

@@ -16,8 +16,9 @@ EVENT_FLOW_DIAGRAM = """
 │  InstallEvent                                                               │
 │    └─ config.json > required_binaries                                       │
 │         └─ BinaryRequestEvent                                               │
-│              └─ on_BinaryRequest__*                                         │
+│              └─ abxpkg BinaryService builtin providers                      │
 │                   └─ BinaryEvent                                            │
+│                        └─ BinaryCacheService / project cache backend        │
 │                                                                             │
 │  CrawlEvent                                                                 │
 │    └─ CrawlSetupEvent                                                       │
@@ -56,13 +57,15 @@ def pluginmap(
     from rich.panel import Panel
     from rich import box
 
-    from archivebox.hooks import (
-        BUILTIN_PLUGINS_DIR,
-        USER_PLUGINS_DIR,
+    from archivebox.plugins.hooks import (
         discover_hooks,
         is_background_hook,
         normalize_hook_event_name,
     )
+    from archivebox.plugins.discovery import (
+        BUILTIN_PLUGINS_DIR,
+        USER_PLUGINS_DIR,
+    )
 
     console = Console()
     prnt = console.print
@@ -73,12 +76,14 @@ def pluginmap(
             "emits": ["BinaryRequestEvent", "BinaryEvent", "ProcessEvent"],
         },
         "BinaryRequestEvent": {
-            "description": "Provider phase. on_BinaryRequest hooks resolve or install requested binaries.",
+            "description": "Binary resolution phase. abxpkg BinaryService resolves or installs requested binaries using built-in providers.",
             "emits": ["BinaryEvent", "ProcessEvent"],
+            "direct_hooks": False,
         },
         "BinaryEvent": {
-            "description": "Resolved binary metadata event. Projected into the DB binary cache.",
+            "description": "Resolved binary metadata event. ArchiveBoxBinaryService projects it into the ArchiveBox DB binary cache.",
             "emits": [],
+            "direct_hooks": False,
         },
         "CrawlEvent": {
             "description": "Root crawl lifecycle event emitted by the runner.",
@@ -138,7 +143,8 @@ def pluginmap(
 
     for event_name, info in event_phases.items():
         hook_event = normalize_hook_event_name(event_name)
-        hooks = discover_hooks(event_name, filter_disabled=not show_disabled)
+        has_direct_hooks = info.get("direct_hooks", True)
+        hooks = discover_hooks(event_name, filter_disabled=not show_disabled) if has_direct_hooks else []
 
         hook_infos = []
         for hook_path in hooks:
@@ -192,8 +198,10 @@ def pluginmap(
         prnt(f"[dim]{info['description']}[/dim]")
         if info["emits"]:
             prnt(f"[dim]Emits: {', '.join(info['emits'])}[/dim]")
-        if not hook_infos:
+        if not hook_infos and has_direct_hooks:
             prnt(f"[dim]No direct on_{hook_event}__* scripts are currently defined for this event family.[/dim]")
+        elif not has_direct_hooks:
+            prnt("[dim]No direct plugin hook family. This event is handled by services.[/dim]")
         prnt()
 
     if not quiet:

archivebox/cli/archivebox_process.py

@@ -30,7 +30,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================

archivebox/cli/archivebox_remove.py

@@ -3,14 +3,16 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox remove"
 
+import time
 from pathlib import Path
 from collections.abc import Iterable
 
 import rich_click as click
 
+from django.db import OperationalError
 from django.db.models import QuerySet
 
-from archivebox.config import DATA_DIR
+from archivebox.config import CONSTANTS
 from archivebox.config.django import setup_django
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.checks import check_data_folder
@@ -21,6 +23,7 @@ from archivebox.misc.logging_util import (
     log_removal_finished,
     TimedProgress,
 )
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
 
 @enforce_types
@@ -31,66 +34,122 @@ def remove(
     after: float | None = None,
     before: float | None = None,
     yes: bool = False,
-    out_dir: Path = DATA_DIR,
-) -> QuerySet:
+    out_dir: Path = CONSTANTS.DATA_DIR,
+    timeout: float | None = None,
+    **kwargs,
+) -> dict[str, object]:
     """Remove the specified URLs from the archive"""
 
     setup_django()
     check_data_folder()
+    timeout = float(timeout) if timeout is not None else None
 
-    from archivebox.cli.archivebox_search import get_snapshots
+    from archivebox.core.models import Snapshot
 
+    filter_kwargs = {
+        **kwargs,
+        "filter_patterns": filter_patterns,
+        "filter_type": filter_type,
+        "after": after,
+        "before": before,
+    }
     pattern_list = list(filter_patterns)
 
     log_list_started(pattern_list or None, filter_type)
     timer = TimedProgress(360, prefix="      ")
     try:
-        snapshots = get_snapshots(
-            snapshots=snapshots,
-            filter_patterns=pattern_list or None,
-            filter_type=filter_type,
-            after=after,
-            before=before,
-        )
+        if snapshots is None:
+            snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
+        # Freeze the target set up-front so a concurrent daemon writing new
+        # snapshots can't extend the deletion under us, and so the cursor isn't
+        # held open across the per-row deletes below.
+        snapshot_pks = list(snapshots.values_list("pk", flat=True))
     finally:
         timer.end()
 
-    if not snapshots.exists():
+    if not snapshot_pks:
         log_removal_finished(0, 0)
         raise SystemExit(1)
 
-    log_list_finished(snapshots)
-    log_removal_started(snapshots, yes=yes)
+    if not yes:
+        log_list_finished(snapshots)
+        log_removal_started(snapshots, yes=False)
 
-    to_remove = snapshots.count()
+    from archivebox.search.query import flush_search_index
 
-    from archivebox.search import flush_search_index
-    from archivebox.core.models import Snapshot
+    started_at = time.monotonic()
+    deadline = started_at + timeout if timeout is not None else None
 
+    # Search-index flush touches a separate backend (FTS / sonic), not the
+    # main index.sqlite3 writer lock, so it's safe to do once up front.
     flush_search_index(snapshots=snapshots)
-    snapshots.delete()
-    all_snapshots = Snapshot.objects.all()
-    log_removal_finished(all_snapshots.count(), to_remove)
 
-    return all_snapshots
+    # Delete one snapshot at a time. Each ``.delete()`` is its own short
+    # Django-atomic block, so the writer lock is released between rows and
+    # an in-flight daemon transaction can interleave instead of deadlocking.
+    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
+    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
+    # rmtree doesn't hold the lock either.
+    #
+    # The SQLite retry wrapper in core/sqlite_backend/base.py re-raises lock
+    # errors when called inside an atomic block (because it can't safely
+    # release+reacquire a transaction), so we wrap each row's delete in our
+    # own retry loop at this outer (non-atomic) level. Each attempt is a
+    # fresh atomic; an exception cleanly rolls it back before we sleep.
+    retry_interval = 1.0
+    deleted_snapshot_pks = []
+    timed_out = False
+    timeout_error = ""
+    for index, pk in enumerate(snapshot_pks):
+        if deadline is not None and time.monotonic() >= deadline:
+            timed_out = True
+            timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining."
+            break
+        while True:
+            try:
+                deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
+                if deleted_count:
+                    deleted_snapshot_pks.append(pk)
+                break
+            except OperationalError as err:
+                if "database is locked" not in str(err):
+                    raise
+                remaining_time = deadline - time.monotonic() if deadline is not None else None
+                if remaining_time is not None and remaining_time <= 0:
+                    timed_out = True
+                    timeout_error = f"Remove timed out after {timeout:g}s while waiting for the database lock."
+                    break
+                time.sleep(min(retry_interval, remaining_time) if remaining_time is not None else retry_interval)
+        if timed_out:
+            break
+
+    all_snapshots = Snapshot.objects.all()
+    remaining_count = all_snapshots.count()
+    deleted_snapshot_id_set = set(deleted_snapshot_pks)
+    remaining_snapshot_pks = [snapshot_id for snapshot_id in snapshot_pks if snapshot_id not in deleted_snapshot_id_set]
+    log_removal_finished(remaining_count, len(deleted_snapshot_pks))
+
+    return {
+        "removed_count": len(deleted_snapshot_pks),
+        "removed_snapshot_ids": [str(snapshot_id) for snapshot_id in deleted_snapshot_pks],
+        "not_removed_count": len(remaining_snapshot_pks),
+        "not_removed_snapshot_ids": [str(snapshot_id) for snapshot_id in remaining_snapshot_pks],
+        "success": not timed_out,
+        "error": timeout_error,
+        "timeout": timeout,
+    }
 
 
 @click.command()
 @click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
-@click.option("--before", type=float, help="Remove only URLs bookmarked before timestamp")
-@click.option("--after", type=float, help="Remove only URLs bookmarked after timestamp")
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(("exact", "substring", "domain", "regex", "tag")),
-    default="exact",
-    help="Type of pattern matching to use when filtering URLs",
-)
-@click.argument("filter_patterns", nargs=-1)
+@click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots")
+@snapshot_filter_options(default_filter_type="exact")
 @docstring(remove.__doc__)
 def main(**kwargs):
     """Remove the specified URLs from the archive"""
-    remove(**kwargs)
+    result = remove(**kwargs)
+    if not result["success"]:
+        raise SystemExit(124)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_run.py

@@ -40,14 +40,23 @@ Examples:
 __package__ = "archivebox.cli"
 __command__ = "archivebox run"
 
-import sys
 import asyncio
+import os
+import signal
+import sys
 from collections import defaultdict
 
 import rich_click as click
 from rich import print as rprint
 
 
+RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
+
+
+def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
+    os._exit(128 + int(sig))
+
+
 def process_stdin_records() -> int:
     """
     Process JSONL records from stdin.
@@ -234,11 +243,17 @@ def process_stdin_records() -> int:
                     crawl_id,
                     snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
                     selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
+                    selected_plugins_are_explicit=False,
                 )
     return 0
 
 
-def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int:
+def run_runner(
+    daemon: bool = False,
+    crawl_id: str | None = None,
+    maintenance_only: bool = False,
+    maintenance_batch_size: int | None = None,
+) -> int:
     """
     Run the background runner loop.
 
@@ -250,11 +265,24 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
     from archivebox.config import CONSTANTS
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Machine, Process
-    from archivebox.services.supervision_service import enter_single_runner_gate
-    from archivebox.services.runner import recover_orchestrator_state, run_pending_crawls
+    from archivebox.core.takeover_util import enter_single_runner_gate, standby_until_foreground_runner_needed
+    from archivebox.core.recovery_util import recover_orchestrator_state
+    from archivebox.services.runner import run_pending_crawls
 
     Machine.current()
     current = Process.current()
+    root_command = current.root
+    if daemon and root_command.process_type in (
+        Process.TypeChoices.SERVER,
+        Process.TypeChoices.ADD,
+        Process.TypeChoices.UPDATE,
+    ):
+        # Server-owned daemon runners are persistent supervisor workers, but
+        # foreground add/update commands are allowed to borrow runner/sonic
+        # leadership without taking down Daphne. Waiting here keeps the worker
+        # on the normal runner path while preventing a server restart loop from
+        # immediately stealing the single-runner gate back from the newer CLI.
+        standby_until_foreground_runner_needed(root_command, data_dir=CONSTANTS.DATA_DIR)
     if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
         current.mark_exited()
         return 0
@@ -264,23 +292,47 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
         from django.utils import timezone
         from archivebox.crawls.models import Crawl
 
-        crawl = Crawl.objects.filter(id=crawl_id, status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED]).first()
-        if crawl is not None and crawl.retry_at is None:
-            crawl.safe_update({"retry_at": timezone.now()}, refresh=False)
+        crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
+        now = timezone.now()
+        # Only re-lease when the row is unscheduled (retry_at IS NULL) or its
+        # existing lease has already expired. A future retry_at means another
+        # worker is already scheduled — don't clobber.
+        if crawl is not None and (crawl.retry_at is None or crawl.retry_at <= now):
+            # extra_filter pins the read-time retry_at so a concurrent worker
+            # that grabbed the lease between our SELECT and UPDATE wins.
+            crawl.safe_update(
+                {"retry_at": now},
+                refresh=False,
+                extra_filter={"retry_at": crawl.retry_at},
+            )
     # Only a foreground `archivebox add` gets the interactive "abort current
     # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
     # orchestrators should shut down immediately and cleanly on the first signal.
     interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
+    if daemon:
+        os.environ[RUNNER_DAEMON_ENV] = "1"
     try:
-        with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
+        with (
+            foreground_shutdown_signals(
+                on_signal=_exit_daemon_runner_on_signal if daemon else None,
+                raise_on_first_signal=not daemon,
+            ),
+            foreground_parent_watchdog(enabled=not daemon),
+        ):
             run_pending_crawls(
                 daemon=daemon,
                 crawl_id=crawl_id,
                 maintenance_only=maintenance_only,
                 interactive_interrupts=interactive_interrupts,
+                **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
             )
         return 0
-    except (KeyboardInterrupt, asyncio.CancelledError):
+    except KeyboardInterrupt:
+        return 0
+    except asyncio.CancelledError as e:
+        if daemon:
+            rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+            return 1
         return 0
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
@@ -297,7 +349,23 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
 @click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
-def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool):
+@click.option(
+    "--maintenance-batch-size",
+    type=int,
+    default=None,
+    hidden=True,
+    help="Limit queued maintenance snapshots claimed per scheduler tick",
+)
+@click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
+def main(
+    daemon: bool,
+    crawl_id: str,
+    snapshot_id: str,
+    binary_id: str,
+    maintenance_only: bool,
+    maintenance_batch_size: int | None,
+    no_stdin: bool,
+):
     """
     Process queued work.
 
@@ -310,6 +378,20 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
     """
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
 
+    if daemon and not snapshot_id and not binary_id and not crawl_id:
+        try:
+            os.environ[RUNNER_DAEMON_ENV] = "1"
+            with (
+                foreground_shutdown_signals(
+                    on_signal=_exit_daemon_runner_on_signal,
+                    raise_on_first_signal=False,
+                ),
+                foreground_parent_watchdog(enabled=False),
+            ):
+                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
+        except KeyboardInterrupt:
+            sys.exit(0)
+
     with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
         if snapshot_id:
             sys.exit(run_snapshot_worker(snapshot_id))
@@ -330,33 +412,49 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
                 sys.exit(1)
 
         if crawl_id:
-            sys.exit(run_runner(daemon=False, crawl_id=crawl_id, maintenance_only=maintenance_only))
+            sys.exit(
+                run_runner(
+                    daemon=False,
+                    crawl_id=crawl_id,
+                    maintenance_only=maintenance_only,
+                    maintenance_batch_size=maintenance_batch_size,
+                ),
+            )
 
         if maintenance_only:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=True))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=True, maintenance_batch_size=maintenance_batch_size))
 
-        if daemon:
-            sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
-
-        if not sys.stdin.isatty():
+        if not no_stdin and not sys.stdin.isatty():
             sys.exit(process_stdin_records())
         else:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
 
 
 def run_snapshot_worker(snapshot_id: str) -> int:
+    from archivebox.config import CONSTANTS
+    from archivebox.core.takeover_util import enter_single_runner_gate
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    from archivebox.machine.models import Process
     from archivebox.core.models import Snapshot
     from archivebox.services.runner import run_due_snapshot
     from django.utils import timezone
 
+    current = Process.current()
+    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
+        current.mark_exited()
+        return 0
+
     snapshot = None
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
-            snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-            if snapshot.retry_at is None:
-                snapshot.update_and_requeue(retry_at=timezone.now())
-            run_due_snapshot(snapshot, lock_seconds=60)
+            for _ in range(10):
+                snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+                if snapshot.retry_at is None:
+                    snapshot.update_and_requeue(retry_at=timezone.now())
+                elif snapshot.retry_at > timezone.now():
+                    break
+                if not run_due_snapshot(snapshot, lock_seconds=60):
+                    break
         return 0
     except KeyboardInterrupt:
         try:
@@ -375,6 +473,10 @@ def run_snapshot_worker(snapshot_id: str) -> int:
 
         traceback.print_exc()
         return 1
+    finally:
+        current.refresh_from_db()
+        if current.status != Process.StatusChoices.EXITED:
+            current.mark_exited()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_schedule.py

@@ -6,7 +6,6 @@ import rich_click as click
 from rich import print
 
 from archivebox.misc.util import enforce_types, docstring
-from archivebox.config.common import get_config
 
 
 @enforce_types
@@ -20,9 +19,8 @@ def schedule(
     every: str | None = None,
     tag: str = "",
     depth: int | str = 0,
-    overwrite: bool = False,
-    update: bool | None = None,
     import_path: str | None = None,
+    config: dict[str, object] | None = None,
 ):
     """Manage database-backed scheduled crawls processed by the crawl runner."""
 
@@ -30,12 +28,10 @@ def schedule(
 
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl, CrawlSchedule
-    from archivebox.crawls.schedule_utils import validate_schedule
+    from archivebox.crawls.schedule_util import validate_schedule
     from archivebox.services.runner import run_pending_crawls
 
-    if update is None:
-        update = not get_config().ONLY_NEW
-
+    config_overrides = dict(config or {})
     depth = int(depth)
     result: dict[str, object] = {
         "created_schedule_ids": [],
@@ -79,10 +75,12 @@ def schedule(
             status=Crawl.StatusChoices.SEALED,
             retry_at=None,
             config={
-                "ONLY_NEW": not update,
-                "OVERWRITE": overwrite,
                 "DEPTH": 0 if is_update_schedule else depth,
                 "SCHEDULE_KIND": "update" if is_update_schedule else "crawl",
+                # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) win over the
+                # template defaults. Anything left unset falls through to the
+                # standard config stack at crawl-resolution time.
+                **config_overrides,
             },
         )
         crawl_schedule = CrawlSchedule.objects.create(
@@ -163,8 +161,13 @@ def schedule(
     default="0",
     help="Recursively archive linked pages up to N hops away",
 )
-@click.option("--overwrite", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
-@click.option("--update", is_flag=True, help="Retry previously failed/skipped URLs when scheduled crawls run")
+@click.option(
+    "--only-new/--no-only-new",
+    "only_new",
+    default=None,
+    help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
+    "Pass --no-only-new to force re-archive on each scheduled run.",
+)
 @click.option("--clear", is_flag=True, help="Disable all currently enabled schedules")
 @click.option("--show", is_flag=True, help="Print all currently enabled schedules")
 @click.option("--foreground", "-f", is_flag=True, help="Run the global crawl runner in the foreground (no crontab required)")
@@ -173,6 +176,9 @@ def schedule(
 @docstring(schedule.__doc__)
 def main(**kwargs):
     """Manage database-backed scheduled crawls processed by the crawl runner."""
+    only_new = kwargs.pop("only_new", None)
+    if only_new is not None:
+        kwargs["config"] = {"ONLY_NEW": bool(only_new)}
     schedule(**kwargs)
 
 

archivebox/cli/archivebox_search.py

@@ -3,255 +3,7 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox search"
 
-import sys
-from pathlib import Path
-from typing import TYPE_CHECKING
-from collections.abc import Callable
-
-import rich_click as click
-
-from django.db.models import Q, QuerySet
-
-from archivebox.config import DATA_DIR
-from archivebox.config.common import get_config
-from archivebox.misc.logging import stderr
-from archivebox.misc.util import enforce_types, docstring
-
-if TYPE_CHECKING:
-    from archivebox.core.models import Snapshot
-
-# Filter types for URL matching
-LINK_FILTERS: dict[str, Callable[[str], Q]] = {
-    "exact": lambda pattern: Q(url=pattern),
-    "substring": lambda pattern: Q(url__icontains=pattern),
-    "regex": lambda pattern: Q(url__iregex=pattern),
-    "domain": lambda pattern: (
-        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
-    ),
-    "tag": lambda pattern: Q(tags__name=pattern),
-    "timestamp": lambda pattern: Q(timestamp=pattern),
-}
-
-STATUS_CHOICES = ["indexed", "archived", "unarchived"]
-
-
-def _apply_pattern_filters(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    filter_patterns: list[str],
-    filter_type: str,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    filter_builder = LINK_FILTERS.get(filter_type)
-    if filter_builder is None:
-        stderr()
-        stderr(f"[X] Got invalid pattern for --filter-type={filter_type}", color="red")
-        raise SystemExit(2)
-
-    query = Q()
-    for pattern in filter_patterns:
-        query |= filter_builder(pattern)
-    return snapshots.filter(query)
-
-
-def _snapshots_to_json(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from archivebox.config import VERSION
-    from archivebox.misc.util import to_json
-
-    config = get_config()
-    main_index_header = (
-        {
-            "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
-            "schema": "archivebox.index.json",
-            "copyright_info": config.FOOTER_INFO,
-            "meta": {
-                "project": "ArchiveBox",
-                "version": VERSION,
-                "git_sha": VERSION,
-                "website": "https://ArchiveBox.io",
-                "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
-                "source": "https://github.com/ArchiveBox/ArchiveBox",
-                "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
-                "dependencies": {},
-            },
-        }
-        if with_headers
-        else {}
-    )
-
-    snapshot_dicts = [snapshot.to_dict(extended=True) for snapshot in snapshots.iterator(chunk_size=500)]
-    output: dict[str, object] | list[dict[str, object]]
-    if with_headers:
-        output = {
-            **main_index_header,
-            "num_links": len(snapshot_dicts),
-            "updated": datetime.now(tz.utc),
-            "last_run_cmd": sys.argv,
-            "links": snapshot_dicts,
-        }
-    else:
-        output = snapshot_dicts
-
-    return to_json(output, indent=4, sort_keys=True)
-
-
-def _snapshots_to_csv(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    cols: list[str],
-    with_headers: bool,
-) -> str:
-    header = ",".join(cols) if with_headers else ""
-    rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.iterator(chunk_size=500)]
-    return "\n".join((header, *rows))
-
-
-def _snapshots_to_html(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from django.template.loader import render_to_string
-
-    from archivebox.config import VERSION
-    from archivebox.config.version import get_COMMIT_HASH
-
-    config = get_config()
-    template = "static_index.html" if with_headers else "minimal_index.html"
-    snapshot_list = list(snapshots.iterator(chunk_size=500))
-
-    return render_to_string(
-        template,
-        {
-            "version": VERSION,
-            "git_sha": get_COMMIT_HASH() or VERSION,
-            "num_links": str(len(snapshot_list)),
-            "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"),
-            "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"),
-            "links": snapshot_list,
-            "FOOTER_INFO": config.FOOTER_INFO,
-        },
-    )
-
-
-def get_snapshots(
-    snapshots: QuerySet["Snapshot", "Snapshot"] | None = None,
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    after: float | None = None,
-    before: float | None = None,
-    out_dir: Path = DATA_DIR,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    """Filter and return Snapshots matching the given criteria."""
-    from archivebox.core.models import Snapshot
-
-    if snapshots is not None:
-        result = snapshots
-    else:
-        result = Snapshot.objects.all()
-
-    if after is not None:
-        result = result.filter(timestamp__gte=after)
-    if before is not None:
-        result = result.filter(timestamp__lt=before)
-    if filter_patterns:
-        result = _apply_pattern_filters(result, filter_patterns, filter_type)
-
-    # Prefetch crawl relationship to avoid N+1 queries when accessing output_dir
-    result = result.select_related("crawl", "crawl__created_by")
-
-    if not result.exists():
-        stderr("[!] No Snapshots matched your filters:", filter_patterns, f"({filter_type})", color="lightyellow")
-
-    return result
-
-
-@enforce_types
-def search(
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    status: str = "indexed",
-    before: float | None = None,
-    after: float | None = None,
-    sort: str | None = None,
-    json: bool = False,
-    html: bool = False,
-    csv: str | None = None,
-    with_headers: bool = False,
-):
-    """List, filter, and export information about archive entries"""
-
-    if with_headers and not (json or html or csv):
-        stderr("[X] --with-headers requires --json, --html or --csv\n", color="red")
-        raise SystemExit(2)
-
-    # Query DB directly - no filesystem scanning
-    snapshots = get_snapshots(
-        filter_patterns=list(filter_patterns) if filter_patterns else None,
-        filter_type=filter_type,
-        before=before,
-        after=after,
-    )
-
-    # Apply status filter
-    if status == "archived":
-        snapshots = snapshots.filter(downloaded_at__isnull=False)
-    elif status == "unarchived":
-        snapshots = snapshots.filter(downloaded_at__isnull=True)
-    # 'indexed' = all snapshots (no filter)
-
-    if sort:
-        snapshots = snapshots.order_by(sort)
-
-    # Export to requested format
-    if json:
-        output = _snapshots_to_json(snapshots, with_headers=with_headers)
-    elif html:
-        output = _snapshots_to_html(snapshots, with_headers=with_headers)
-    elif csv:
-        output = _snapshots_to_csv(snapshots, cols=csv.split(","), with_headers=with_headers)
-    else:
-        from archivebox.misc.logging_util import printable_folders
-
-        # Convert to dict for printable_folders
-        folders: dict[str, Snapshot | None] = {str(snapshot.output_dir): snapshot for snapshot in snapshots}
-        output = printable_folders(folders, with_headers)
-
-    # Structured exports must be written directly to stdout.
-    # rich.print() reflows long lines to console width, which corrupts JSON/CSV/HTML output.
-    sys.stdout.write(output)
-    if not output.endswith("\n"):
-        sys.stdout.write("\n")
-    return output
-
-
-@click.command()
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(["search", *LINK_FILTERS.keys()]),
-    default="substring",
-    help="Pattern matching type for filtering URLs",
-)
-@click.option("--status", "-s", type=click.Choice(STATUS_CHOICES), default="indexed", help="List snapshots with the given status")
-@click.option("--before", "-b", type=float, help="List snapshots bookmarked before the given UNIX timestamp")
-@click.option("--after", "-a", type=float, help="List snapshots bookmarked after the given UNIX timestamp")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--json", "-J", is_flag=True, help="Print output in JSON format")
-@click.option("--html", "-M", is_flag=True, help="Print output in HTML format (suitable for viewing statically without a server)")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: created_at,url,title")
-@click.option("--with-headers", "-H", is_flag=True, help="Include extra CSV/HTML headers in the output")
-@click.help_option("--help", "-h")
-@click.argument("filter_patterns", nargs=-1)
-@docstring(search.__doc__)
-def main(**kwargs):
-    return search(**kwargs)
+from archivebox.cli.archivebox_list import main
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_server.py

@@ -12,14 +12,222 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print
 
+from archivebox.config import CONSTANTS
 from archivebox.misc.util import docstring, enforce_types
 
 
+import re as _re
+
+_IPV4_RE = _re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
+_IPV6_CHARS_RE = _re.compile(r"^[0-9a-fA-F:.]+$")
+_LOCAL_BIND_HOSTS = frozenset({"0.0.0.0", "::", "::0", "127.0.0.1", "::1"})
+
+
+def _is_ipv4_literal(host: str) -> bool:
+    return bool(_IPV4_RE.match(host))
+
+
+def _is_ipv6_literal(host: str) -> bool:
+    # Bracketed (e.g. ``[2001:db8::1]``) or bare form. Require at least two
+    # colons so we don't catch random strings with one ``:``.
+    stripped = host.strip("[]")
+    return stripped.count(":") >= 2 and bool(_IPV6_CHARS_RE.match(stripped))
+
+
+def _bind_host_looks_like_ip(host: str) -> bool:
+    if not host or host in _LOCAL_BIND_HOSTS:
+        return False
+    return _is_ipv4_literal(host) or _is_ipv6_literal(host)
+
+
+def _split_bind_spec(spec: str) -> tuple[str, str]:
+    """Split a ``host:port`` / ``host`` / ``port`` spec into ``(host, port)``.
+
+    The empty strings stand in for "not provided"; the caller fills in
+    defaults. Bracketed IPv6 literals like ``[::1]:8000`` are handled.
+    """
+    spec = (spec or "").strip()
+    if not spec:
+        return "", ""
+    if spec.startswith("["):
+        # Bracketed IPv6: ``[::1]`` or ``[::1]:8000``
+        end = spec.find("]")
+        if end == -1:
+            return spec, ""  # malformed; let validator reject it
+        host = spec[: end + 1]
+        rest = spec[end + 1 :]
+        if rest.startswith(":"):
+            return host, rest[1:]
+        return host, ""
+    if ":" in spec:
+        host, _, port = spec.rpartition(":")
+        return host, port
+    # Bare token: digits = port, anything else = host
+    if spec.isdigit():
+        return "", spec
+    return spec, ""
+
+
+def _parse_and_validate_bind_spec(spec: str) -> tuple[str, str]:
+    """Resolve a CLI/config bind spec to ``(host, port)`` or hard-error.
+
+    Accepts only IP literals (v4 or v6) or the special string ``localhost``
+    (normalized to ``127.0.0.1``). Bare hostnames are rejected because the
+    bind address feeds Daphne, which has to listen on a numeric address;
+    public hostnames belong in ``BASE_URL`` instead. Empty values fall back
+    to ``127.0.0.1`` / ``8000``.
+    """
+    raw_host, raw_port = _split_bind_spec(spec)
+    host = raw_host.strip()
+    port = (raw_port or "").strip() or "8000"
+
+    if host == "" or host.lower() == "localhost":
+        host = "127.0.0.1"
+    elif _is_ipv4_literal(host) or _is_ipv6_literal(host):
+        pass
+    else:
+        print(
+            f"[red][X] Invalid BIND_ADDR host {host!r}: must be an IP literal or 'localhost'.[/red]",
+        )
+        print(
+            "[red]    Hostnames like archive.example.com are not valid bind addresses — Daphne[/red]",
+        )
+        print(
+            "[red]    listens on numeric addresses only. Bind to 0.0.0.0 and set BASE_URL instead:[/red]",
+        )
+        print(
+            f"[red]      BASE_URL=https://{host} archivebox server 0.0.0.0:{port}[/red]",
+        )
+        sys.exit(1)
+
+    try:
+        port_int = int(port)
+    except ValueError:
+        print(f"[red][X] Invalid BIND_ADDR port {port!r}: must be an integer 1-65535.[/red]")
+        sys.exit(1)
+    if not (0 < port_int < 65536):
+        print(f"[red][X] Invalid BIND_ADDR port {port_int}: must be 1-65535.[/red]")
+        sys.exit(1)
+
+    return host, port
+
+
+def _print_server_startup_warnings(config, host: str, port: str) -> None:
+    """Print startup-time security / routing warnings for the server command.
+
+    Runs only from ``archivebox server`` so other entry points (manage shell,
+    plugin lookups, etc.) don't repeat this banner on every config load.
+    """
+    if config.IS_LOWER_SECURITY_MODE:
+        print(
+            f"[yellow][!] WARNING: ArchiveBox is running with SERVER_SECURITY_MODE={config.SERVER_SECURITY_MODE}[/yellow]",
+        )
+        print("[yellow]    Archived pages may share an origin with privileged app routes in this mode.[/yellow]")
+        print("[yellow]    To switch to the safer isolated setup:[/yellow]")
+        print("[yellow]      1. Set SERVER_SECURITY_MODE=safe-subdomains-fullreplay[/yellow]")
+        print("[yellow]      2. Point *.archivebox.localhost (or your chosen base domain) at this server[/yellow]")
+        print(
+            "[yellow]      3. Configure wildcard DNS/TLS or your reverse proxy so admin., web., api., and snapshot subdomains resolve[/yellow]",
+        )
+        print()
+
+    # ``config.BASE_URL`` is the merged value (env > Machine.config > file >
+    # default), which is what the running server will actually use. Earlier we
+    # gated the "BASE_URL not set" warning on ``os.environ["BASE_URL"]`` alone,
+    # which fired noisily when the user pinned BASE_URL via Machine.config /
+    # ArchiveBox.conf instead of via env.
+    base_url = (config.BASE_URL or "").strip()
+    if base_url:
+        # BASE_URL is pinned. The only thing left to surface is a port
+        # mismatch — bind port ≠ BASE_URL's explicit port usually means the
+        # operator started the server with the wrong ``archivebox server PORT``
+        # argument (or forgot to update one side after moving the listener).
+        # A reverse-proxy setup typically omits the port in BASE_URL
+        # (``https://archive.example.com``), so we only warn when BASE_URL
+        # carries an explicit port — otherwise we'd nag every proxy deployment.
+        from urllib.parse import urlparse
+
+        try:
+            base_port = urlparse(base_url).port
+        except (ValueError, TypeError):
+            base_port = None
+        if base_port is not None and str(base_port) != str(port):
+            print(
+                f"[yellow][!] BASE_URL ({base_url}) port {base_port} does not match the port the server is running on ({port}). "
+                "Make sure this is intentional![/yellow]",
+            )
+            print()
+        return
+
+    # If the user is upgrading from 0.7.3 and already had
+    # CSRF_TRUSTED_ORIGINS set, get_base_url() will silently use that as the
+    # implicit BASE_URL. Surface what we picked so the user knows where their
+    # links / redirects are going — and tell them how to make it explicit.
+    from archivebox.core.routes_util import derive_base_url_from_csrf
+
+    csrf_derived = derive_base_url_from_csrf(config)
+    if csrf_derived:
+        print(
+            f"[yellow][!] BASE_URL is not set; auto-derived [bold]{csrf_derived}[/bold] from a single CSRF_TRUSTED_ORIGINS entry.[/yellow]",
+        )
+        print(
+            "[yellow]    Links / redirects / cookies will use that origin. To silence this hint, set BASE_URL[/yellow]",
+        )
+        print(
+            f"[yellow]    explicitly: [bold]BASE_URL={csrf_derived}[/bold] (matches your existing CSRF_TRUSTED_ORIGINS).[/yellow]",
+        )
+        print()
+        return
+
+    # BASE_URL was not set explicitly. The routes_util derivation gives one of
+    # three results, with very different risk profiles — show a tailored hint
+    # so new users coming from the 0.7.x single-domain world know whether the
+    # default is fine for them or needs attention.
+    if _bind_host_looks_like_ip(host):
+        # Real IP literal: subdomain routing can't work, URLs leak the IP.
+        # This is the most urgent case.
+        print(
+            f"[yellow][!] WARNING: BASE_URL is not set and BIND_ADDR resolves to an IP literal ({host}).[/yellow]",
+        )
+        print(
+            "[yellow]    Snapshot / admin / api URLs will be generated with the IP, and subdomain[/yellow]",
+        )
+        print(
+            "[yellow]    routing cannot work against an IP address. Set BASE_URL explicitly, e.g.[/yellow]",
+        )
+        print(
+            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
+        )
+        if config.USES_SUBDOMAIN_ROUTING:
+            print(
+                "[yellow]    Or switch SERVER_SECURITY_MODE to a one-domain mode if you can't run a hostname.[/yellow]",
+            )
+        print()
+    else:
+        # Loopback / wildcard bind. The routes_util default of
+        # http://archivebox.localhost:PORT works in a browser on the same
+        # machine, but anything else (reverse proxy, k8s ingress, LAN client)
+        # needs BASE_URL set. (Real hostnames can't reach this branch — the
+        # bind validator rejects them upfront.)
+        print(
+            "[yellow][!] BASE_URL is not set. Generated URLs will fall back to http://archivebox.localhost:<port>.[/yellow]",
+        )
+        print(
+            "[yellow]    That's fine for local browsing on this machine. Set BASE_URL when running behind[/yellow]",
+        )
+        print(
+            "[yellow]    a reverse proxy / ingress / public hostname, e.g.[/yellow]",
+        )
+        print(
+            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
+        )
+        print()
+
+
 @enforce_types
 def server(
     runserver_args: Iterable[str] | None = None,
     reload: bool = False,
-    init: bool = False,
     debug: bool = False,
     daemonize: bool = False,
     nothreading: bool = False,
@@ -30,12 +238,6 @@ def server(
     config = get_config()
     runserver_args = list(runserver_args or (config.BIND_ADDR,))
 
-    if init:
-        from archivebox.cli.archivebox_init import init as archivebox_init
-
-        archivebox_init(quick=True)
-        print()
-
     run_in_debug = config.DEBUG or debug or reload
     if debug or reload:
         os.environ["DEBUG"] = "True"
@@ -50,24 +252,14 @@ def server(
         print("      [green]archivebox manage createsuperuser[/green]")
         print()
 
-    host = "127.0.0.1"
-    port = "8000"
-
-    try:
-        host_and_port = [arg for arg in runserver_args if arg.replace(".", "").replace(":", "").isdigit()][0]
-        if ":" in host_and_port:
-            host, port = host_and_port.split(":")
-        else:
-            if "." in host_and_port:
-                host = host_and_port
-            else:
-                port = host_and_port
-    except IndexError:
-        pass
+    # First non-empty positional arg is the bind spec; otherwise inherit from
+    # config (which defaults to "127.0.0.1:8000"). _parse_and_validate_bind_spec
+    # hard-errors on hostnames so the rest of the server can assume a numeric
+    # bind host.
+    bind_spec = next((arg for arg in runserver_args if arg), "")
+    host, port = _parse_and_validate_bind_spec(bind_spec)
 
     if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1":
-        from archivebox.config import CONSTANTS
-
         log_path = CONSTANTS.LOGS_DIR / "server.log"
         log_path.parent.mkdir(parents=True, exist_ok=True)
         daemon_env = os.environ.copy()
@@ -105,9 +297,10 @@ def server(
         return
 
     os.environ["BIND_ADDR"] = f"{host}:{port}"
-    from archivebox.core.host_utils import build_admin_url
+    from archivebox.core.routes_util import get_base_url
 
-    admin_url = build_admin_url("/admin/")
+    base_url = get_base_url().rstrip("/")
+    admin_url = f"{base_url}/admin/"
 
     from archivebox.workers.supervisord_util import (
         active_supervisord_runtime_components,
@@ -117,9 +310,10 @@ def server(
         is_port_in_use,
     )
     from archivebox.machine.models import Process
-    from archivebox.services.supervision_service import (
+    from archivebox.core.takeover_util import (
         command_owns_runtime_stack,
         current_command,
+        foreground_runner_owner,
         runtime_stack_owner,
         standby_until_runtime_stack_needed,
     )
@@ -130,29 +324,41 @@ def server(
     else:
         print("[green][+] Starting ArchiveBox webserver...[/green]")
     print(
-        f"    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
+        f"    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [dim]BIND_ADDR[/dim] [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
     )
     print(
-        f"    [green]>[/green] Log in to ArchiveBox Admin UI on [deep_sky_blue3][link={admin_url}]{admin_url}[/link][/deep_sky_blue3]",
+        f"    [green]>[/green] Log in to ArchiveBox Admin UI on [dim]BASE_URL [/dim] [deep_sky_blue3][link={admin_url}]{admin_url}[/link][/deep_sky_blue3]",
     )
     print("    > Writing ArchiveBox error log to ./logs/errors.log")
     print()
+
+    # Reload config after we've set os.environ["BIND_ADDR"] above so the
+    # security-mode + base-url warnings see the effective values.
+    runtime_config = get_config()
+    _print_server_startup_warnings(runtime_config, host, port)
     bind_url = f"http://{host}:{port}"
-    command = current_command(Process.TypeChoices.SERVER, data_dir=config.DATA_DIR, url=bind_url)
+    command = current_command(Process.TypeChoices.SERVER, data_dir=CONSTANTS.DATA_DIR, url=bind_url)
 
     def still_owns_runtime_stack() -> bool:
         from django.db import connections
 
         try:
-            return command_owns_runtime_stack(command, data_dir=config.DATA_DIR)
+            return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
         finally:
             connections.close_all()
 
+    shutdown_state = None
     try:
-        with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"):
+        with (
+            foreground_shutdown_signals() as shutdown_state,
+            foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"),
+        ):
             while True:
-                standby_result = standby_until_runtime_stack_needed(command, data_dir=config.DATA_DIR)
-                older_owner = runtime_stack_owner(data_dir=config.DATA_DIR, exclude_id=command.id)
+                standby_result = standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id) or foreground_runner_owner(
+                    data_dir=CONSTANTS.DATA_DIR,
+                    exclude_id=command.id,
+                )
                 takeover_components = active_supervisord_runtime_components(config=config)
                 if older_owner and takeover_components:
                     print(
@@ -187,7 +393,8 @@ def server(
     except KeyboardInterrupt:
         pass
     finally:
-        command.mark_exited()
+        if not shutdown_state or not shutdown_state.signal_name:
+            command.mark_exited()
     print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")
 
 
@@ -196,7 +403,6 @@ def server(
 @click.option("--reload", is_flag=True, help="Enable auto-reloading when code or templates change")
 @click.option("--debug", is_flag=True, help="Enable DEBUG=True mode with more verbose errors")
 @click.option("--nothreading", is_flag=True, help="Force runserver to run in single-threaded mode")
-@click.option("--init", is_flag=True, help="Run a full archivebox init/upgrade before starting the server")
 @click.option("--daemonize", is_flag=True, help="Run the server in the background as a daemon")
 @docstring(server.__doc__)
 def main(**kwargs):

archivebox/cli/archivebox_shell.py

@@ -12,9 +12,10 @@ from archivebox.misc.util import docstring
 def shell(args: Iterable[str] = ()) -> None:
     """Enter an interactive ArchiveBox Django shell"""
 
-    from django.core.management import call_command
+    from django.core.management import call_command, get_commands
 
-    call_command("shell_plus", *args)
+    shell_command = "shell_plus" if "shell_plus" in get_commands() else "shell"
+    call_command(shell_command, *args)
 
 
 @click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))

archivebox/cli/archivebox_snapshot.py

@@ -35,9 +35,10 @@ from collections.abc import Iterable
 
 import rich_click as click
 from rich import print as rprint
-from django.db.models import Case, IntegerField, Q, QuerySet, When
+from django.db.models import QuerySet
 
-from archivebox.cli.cli_utils import apply_filters
+SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp")
+SNAPSHOT_LIST_CHUNK_SIZE = 100
 
 
 # =============================================================================
@@ -66,7 +67,7 @@ def create_snapshots(
         TYPE_SNAPSHOT,
         TYPE_CRAWL,
     )
-    from archivebox.misc.util import validate_url_length
+    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot
     from archivebox.crawls.models import Crawl
@@ -111,7 +112,7 @@ def create_snapshots(
                 # Create snapshots for each URL in the crawl
                 for url in crawl.get_urls_list():
                     try:
-                        validate_url_length(url)
+                        validate_url(url)
                     except ValueError as err:
                         rprint(f"[red]Error creating snapshot: {err}[/red]", file=sys.stderr)
                         continue
@@ -134,7 +135,7 @@ def create_snapshots(
             elif record_type == TYPE_SNAPSHOT or record.get("url"):
                 # Input is a Snapshot or plain URL
                 if record.get("url"):
-                    validate_url_length(str(record["url"]))
+                    validate_url(str(record["url"]))
                 if tag and not record.get("tags"):
                     record["tags"] = tag
                 if status:
@@ -178,94 +179,63 @@ def create_snapshots(
 # =============================================================================
 
 
+def snapshot_filter_options(*, default_filter_type: str):
+    def decorate(func):
+        for decorator in reversed(
+            (
+                click.option("--status", "-s", help="Filter by status (queued, started, sealed)"),
+                click.option("--url__icontains", help="Filter by URL contains"),
+                click.option("--url__istartswith", help="Filter by URL starts with"),
+                click.option("--tag", "-t", help="Filter by tag name"),
+                click.option("--crawl-id", help="Filter by crawl ID"),
+                click.option("--limit", "-n", type=int, help="Limit number of results"),
+                click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at"),
+                click.option("--search", help="Search mode to use for positional query"),
+                click.option("--before", type=float, help="Only snapshots bookmarked before timestamp"),
+                click.option("--after", type=float, help="Only snapshots bookmarked after timestamp"),
+                click.option(
+                    "--filter-type",
+                    "-f",
+                    type=click.Choice(SNAPSHOT_FILTER_TYPE_CHOICES),
+                    default=default_filter_type,
+                    help="Type of pattern matching to use for positional filters",
+                ),
+                click.argument("filter_patterns", nargs=-1),
+            ),
+        ):
+            func = decorator(func)
+        return func
+
+    return decorate
+
+
+def snapshot_output_options(func):
+    for decorator in reversed(
+        (
+            click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title"),
+            click.option("--json", "as_json", is_flag=True, help="Print output as a JSON array"),
+            click.option("--html", "as_html", is_flag=True, help="Print output as HTML"),
+            click.option("--with-headers", is_flag=True, help="Include column headers in structured output"),
+        ),
+    ):
+        func = decorator(func)
+    return func
+
+
 def build_snapshot_queryset(
-    *,
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    sort: str | None = None,
-    search: str | None = None,
-    query: str | None = None,
-    limit: int | None = None,
+    **kwargs,
 ) -> QuerySet:
     from archivebox.core.models import Snapshot
-    from archivebox.search import (
-        get_default_search_mode,
-        get_search_mode,
-        prioritize_metadata_matches,
-        query_search_index,
-    )
 
-    queryset = Snapshot.objects.order_by("-created_at")
-    queryset = apply_filters(
-        queryset,
-        {
-            "status": status,
-            "url__icontains": url__icontains,
-            "url__istartswith": url__istartswith,
-            "crawl_id": crawl_id,
-        },
-    )
-
-    if tag:
-        queryset = queryset.filter(tags__name__iexact=tag)
-
-    query = (query or "").strip()
-    if query:
-        metadata_qs = queryset.filter(
-            Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query),
-        )
-        requested_search_mode = (search or "").strip().lower()
-        if requested_search_mode == "content":
-            requested_search_mode = "contents"
-        search_mode = get_default_search_mode() if not requested_search_mode else get_search_mode(requested_search_mode)
-
-        if search_mode == "meta":
-            queryset = metadata_qs
-        elif limit and len(list(metadata_qs.values_list("pk", flat=True).distinct()[:limit])) >= limit:
-            queryset = metadata_qs
-        else:
-            try:
-                deep_qsearch = None
-                if search_mode == "deep":
-                    qsearch = query_search_index(query, search_mode="contents", max_results=limit)
-                    deep_qsearch = query_search_index(query, search_mode="deep", max_results=limit)
-                else:
-                    qsearch = query_search_index(query, search_mode=search_mode, max_results=limit)
-                queryset = prioritize_metadata_matches(
-                    queryset,
-                    metadata_qs,
-                    qsearch,
-                    deep_queryset=deep_qsearch,
-                    ordering=("-created_at",) if not sort else None,
-                )
-            except Exception as err:
-                rprint(
-                    f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
-                    file=sys.stderr,
-                )
-                queryset = metadata_qs
-
-    if sort:
-        queryset = queryset.order_by(sort)
-
-    return queryset
+    return Snapshot.objects.order_by("-created_at").search(**kwargs)
 
 
 def list_snapshots(
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
     csv: str | None = None,
+    as_json: bool = False,
+    as_html: bool = False,
     with_headers: bool = False,
-    search: str | None = None,
-    query: str | None = None,
+    **kwargs,
 ) -> int:
     """
     List Snapshots as JSONL with optional filters.
@@ -274,96 +244,71 @@ def list_snapshots(
         0: Success (even if no results)
     """
     from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import Snapshot
 
-    if with_headers and not csv:
-        rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr)
+    output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
+    if output_formats > 1:
+        rprint("[red]Choose only one output format: --csv, --json, or --html[/red]", file=sys.stderr)
+        return 2
+    if with_headers and not output_formats:
+        rprint("[red]--with-headers requires --csv, --json, or --html[/red]", file=sys.stderr)
         return 2
 
-    is_tty = sys.stdout.isatty() and not csv
+    is_tty = sys.stdout.isatty() and not output_formats
 
-    queryset = build_snapshot_queryset(
-        status=status,
-        url__icontains=url__icontains,
-        url__istartswith=url__istartswith,
-        tag=tag,
-        crawl_id=crawl_id,
-        sort=sort,
-        search=search,
-        query=query,
-        limit=limit,
-    )
-
-    if not is_tty:
-        if limit:
-            limited_ids = list(queryset.values_list("id", flat=True)[:limit])
-            preserved_order = Case(
-                *(When(id=snapshot_id, then=position) for position, snapshot_id in enumerate(limited_ids)),
-                output_field=IntegerField(),
-            )
-            queryset = Snapshot.objects.filter(id__in=limited_ids).order_by(preserved_order)
-        queryset = queryset.prefetch_related("tags")
-    elif limit:
-        queryset = queryset[:limit]
+    try:
+        queryset = build_snapshot_queryset(**kwargs)
+    except ValueError as err:
+        rprint(f"[red]{err}[/red]", file=sys.stderr)
+        return 2
 
     count = 0
+    if as_json:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_json(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    if as_html:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_html(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
     if csv:
         cols = [col.strip() for col in csv.split(",") if col.strip()]
         if not cols:
             rprint("[red]No CSV columns provided[/red]", file=sys.stderr)
             return 2
-        rows: list[str] = []
         if with_headers:
-            rows.append(",".join(cols))
-        simple_cols = {
-            "id",
-            "crawl_id",
-            "url",
-            "title",
-            "timestamp",
-            "depth",
-            "status",
-            "fs_version",
-            "bookmarked_at",
-            "created_at",
-            "modified_at",
-            "retry_at",
-            "downloaded_at",
-        }
-        from archivebox.misc.util import to_json
-
-        for snapshot in queryset.iterator(chunk_size=500):
-            if set(cols).issubset(simple_cols):
-                rows.append(
-                    ",".join(
-                        to_json(
-                            value.isoformat() if hasattr((value := getattr(snapshot, col, "")), "isoformat") else value,
-                            indent=None,
-                        )
-                        for col in cols
-                    ),
-                )
-            else:
-                rows.append(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write(",".join(cols))
+            sys.stdout.write("\n")
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+            sys.stdout.write(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write("\n")
             count += 1
-        output = "\n".join(rows)
-        if output:
-            sys.stdout.write(output)
-            if not output.endswith("\n"):
-                sys.stdout.write("\n")
         rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
         return 0
 
-    for snapshot in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(snapshot.status, "dim")
-            rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
-        else:
+    if not is_tty:
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
             write_record(snapshot.to_json())
+            count += 1
+        rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    for snapshot in queryset.iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+        status_color = {
+            "queued": "yellow",
+            "started": "blue",
+            "sealed": "green",
+        }.get(snapshot.status, "dim")
+        rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
         count += 1
 
     rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
@@ -430,10 +375,10 @@ def update_snapshots(
 
                 tag_obj, _ = Tag.objects.get_or_create(name=tag)
                 snapshot.tags.add(tag_obj)
-                snapshot.save(update_fields=["modified_at"])
+                snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
 
             if not status and not tag:
-                snapshot.save(update_fields=["modified_at"])
+                snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
             updated_count += 1
 
             if not is_tty:
@@ -522,46 +467,11 @@ def create_cmd(urls: tuple, tag: str, status: str, depth: int):
 
 
 @main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def list_cmd(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-):
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def list_cmd(**kwargs):
     """List Snapshots as JSONL."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))
 
 
 @main.command("update")

archivebox/cli/archivebox_snapshot_compat.py

@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox snapshot"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_snapshot import create_snapshots
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--depth", "-d", type=int, default=0, help="Crawl depth (default: 0)")
-@click.argument("urls", nargs=-1)
-def main(tag: str, status: str, depth: int, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox snapshot URL...` entrypoint."""
-    sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth))
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_status.py

@@ -8,7 +8,7 @@ import rich_click as click
 from rich import print
 
 from archivebox.misc.util import enforce_types, docstring
-from archivebox.config import DATA_DIR, CONSTANTS
+from archivebox.config import CONSTANTS
 from archivebox.config.common import get_config
 from archivebox.misc.system import get_dir_size
 from archivebox.misc.logging_util import printable_filesize
@@ -18,7 +18,7 @@ MAX_STATUS_FS_DIR_SCAN = 5000
 
 
 @enforce_types
-def status(out_dir: Path = DATA_DIR) -> None:
+def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
     """Print out some info and statistics about the archive collection"""
 
     from django.contrib.auth import get_user_model
@@ -38,7 +38,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
 
     snapshots_qs = Snapshot.objects.all()
     num_sql_links = snapshots_qs.count()
-    archive_dir = config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
     legacy_snapshot_dirs = []
     if archive_dir.exists():
         legacy_snapshot_dirs = [
@@ -48,7 +48,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
     print(f"    > JSON Link Details: {len(legacy_snapshot_dirs)} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)")
     print()
     print("[green]\\[*] Scanning archive data directories...[/green]")
-    users_dir = config.USERS_DIR
+    users_dir = CONSTANTS.USERS_DIR
     scan_roots = [root for root in (archive_dir, users_dir) if root.exists()]
     scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(archive_dir)
     print(f"[yellow]   {scan_roots_display}[/yellow]")
@@ -61,7 +61,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
             num_dirs += root_dirs
             num_files += root_files
     else:
-        num_bytes = ArchiveResult.objects.aggregate(total=Coalesce(Sum("output_size"), 0))["total"] or 0
+        num_bytes = snapshots_qs.aggregate(total=Coalesce(Sum("output_size"), 0))["total"] or 0
         num_dirs = 0
         num_files = ArchiveResult.objects.exclude(output_files__in=["", "{}"]).count()
     size = printable_filesize(num_bytes)
@@ -105,7 +105,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
 
     if num_indexed:
         print("    [violet]Hint:[/violet] You can list snapshots by status like so:")
-        print("        [green]archivebox list --status=<status>  (e.g. archived, queued, etc.)[/green]")
+        print("        [green]archivebox list --status=<status>  (e.g. sealed, queued, etc.)[/green]")
 
     if orphaned_dirs:
         print("    [violet]Hint:[/violet] To automatically import orphaned data directories into the main index, run:")

archivebox/cli/archivebox_tag.py

@@ -36,7 +36,7 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================

archivebox/cli/archivebox_update.py

@@ -15,6 +15,7 @@ from pathlib import Path
 import rich_click as click
 
 from archivebox.misc.util import enforce_types, docstring
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
@@ -32,66 +33,36 @@ def _get_snapshot_crawl(snapshot: Snapshot) -> Crawl | None:
 
 
 def _get_search_indexing_plugins() -> list[str]:
-    from abx_dl.models import discover_plugins
-    from archivebox.hooks import get_search_backends
+    from archivebox.config.common import get_config
+    from archivebox.plugins.hooks import discover_hooks
+    from archivebox.plugins.discovery import get_search_backends
 
     available_backends = set(get_search_backends())
-    plugins = discover_plugins()
     return sorted(
         plugin_name
-        for plugin_name, plugin in plugins.items()
-        if plugin_name.startswith("search_backend_")
-        and plugin_name.removeprefix("search_backend_") in available_backends
-        and any("Snapshot" in hook.name and "index" in hook.name.lower() for hook in plugin.hooks)
+        for plugin_name in {
+            hook.parent.name
+            for hook in discover_hooks("Snapshot", config=get_config())
+            if hook.parent.name.startswith("search_backend_") and "index" in hook.name.lower()
+        }
+        if plugin_name.startswith("search_backend_") and plugin_name.removeprefix("search_backend_") in available_backends
     )
 
 
 def _build_filtered_snapshots_queryset(
-    *,
-    filter_patterns: Iterable[str],
-    filter_type: str,
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
-    search: str | None = None,
-    before: float | None = None,
-    after: float | None = None,
-    resume: str | None = None,
+    **kwargs,
 ):
-    from datetime import datetime
+    from archivebox.core.models import Snapshot
     from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
-    filter_patterns = tuple(filter_patterns)
-    snapshots = build_snapshot_queryset(
-        status=status,
-        url__icontains=url__icontains,
-        url__istartswith=url__istartswith,
-        tag=tag,
-        crawl_id=crawl_id,
-        sort=sort,
-        search=search,
-        query=" ".join(filter_patterns) if search else None,
-    )
-
-    if filter_patterns and not search:
-        snapshots = snapshots.filter_by_patterns(list(filter_patterns), filter_type)
-
-    if before:
-        snapshots = snapshots.filter(bookmarked_at__lt=datetime.fromtimestamp(before))
-    if after:
-        snapshots = snapshots.filter(bookmarked_at__gt=datetime.fromtimestamp(after))
-    if resume:
-        snapshots = snapshots.filter(timestamp__lte=resume)
-    if not sort:
-        snapshots = snapshots.order_by("-timestamp")
+    limit = kwargs.pop("limit", None)
+    snapshots = build_snapshot_queryset(**kwargs)
+    if kwargs.get("resume"):
+        snapshots = snapshots.filter(timestamp__lte=kwargs["resume"])
     snapshots = snapshots.select_related("crawl")
-    if limit:
-        limited_ids = list(snapshots.values_list("id", flat=True)[:limit])
-        snapshots = snapshots.model.objects.filter(id__in=limited_ids).select_related("crawl")
+    if limit is not None and limit > 0:
+        snapshot_ids = list(snapshots.values_list("id", flat=True)[:limit])
+        snapshots = Snapshot.objects.filter(id__in=snapshot_ids).select_related("crawl")
 
     return snapshots
 
@@ -105,9 +76,17 @@ def reindex_snapshots(
     wait_for_turn=None,
 ) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
+    from archivebox.core.models import ArchiveResult
+    from abx_dl.models import discover_plugins
 
-    stats: dict[str, Any] = {"processed": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
+    stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
     records: list[dict[str, str]] = []
+    plugins_by_name = discover_plugins(runtime="archivebox")
+    required_hooks_by_plugin = {
+        plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
+        for plugin_name in search_plugins
+        if plugin_name in plugins_by_name
+    }
 
     total = snapshots.count()
     print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}")
@@ -118,6 +97,31 @@ def reindex_snapshots(
         if wait_for_turn:
             wait_for_turn()
         batch_records = list(records)
+        snapshot_ids = {record["snapshot_id"] for record in batch_records}
+        plugin_names = {record["plugin"] for record in batch_records}
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=snapshot_ids,
+                plugin__in=plugin_names,
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        records_to_queue = []
+        for record in batch_records:
+            snapshot_id = record["snapshot_id"]
+            plugin_name = record["plugin"]
+            required_hooks = required_hooks_by_plugin.get(plugin_name, frozenset())
+            if required_hooks and all((snapshot_id, plugin_name, hook_name) in queued_rows for hook_name in required_hooks):
+                stats["skipped_queued"] += 1
+                continue
+            records_to_queue.append(record)
+        if not records_to_queue:
+            print(
+                f"    [{stats['processed']}/{total}] Already queued {len(batch_records)} index jobs",
+            )
+            records.clear()
+            return
         # `archivebox update --index-only` intentionally breaks the usual
         # "runner discovers work" rule by inserting synthetic queued
         # ArchiveResult rows for search backends. run_plugins() keeps this as
@@ -127,15 +131,17 @@ def reindex_snapshots(
         # finish.
         exit_code = run_plugins(
             args=(),
-            records=batch_records,
+            records=records_to_queue,
             wait=False,
             emit_results=False,
             show_progress=False,
+            preserve_queued=True,
         )
         if exit_code != 0:
             raise SystemExit(exit_code)
+        stats["queued"] += len(records_to_queue)
         print(
-            f"    [{stats['processed']}/{total}] Queued {len(batch_records)} index jobs for orchestrator",
+            f"    [{stats['processed']}/{total}] Queued {len(records_to_queue)} index jobs for orchestrator",
         )
         records.clear()
 
@@ -156,7 +162,7 @@ def reindex_snapshots(
                         "plugin": plugin_name,
                     },
                 )
-                stats["queued"] += 1
+                stats["requested"] += 1
             if len(records) >= batch_size:
                 run_batch()
         except KeyboardInterrupt as err:
@@ -218,28 +224,43 @@ def update(
 
     from archivebox.machine.models import Process
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals, raise_if_shutdown_requested
-    from archivebox.services.supervision_service import (
-        command_owns_runtime_stack,
+    from archivebox.core.takeover_util import (
+        command_owns_foreground_runner,
         current_command,
         ensure_daemon_stack,
-        standby_until_runtime_stack_needed,
+        foreground_runner_owner,
+        standby_until_foreground_runner_needed,
     )
-    from archivebox.workers.supervisord_util import run_runner_worker, stop_existing_supervisord_process, stop_own_supervisord_process
+    from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
 
     command = current_command(Process.TypeChoices.UPDATE, data_dir=CONSTANTS.DATA_DIR)
 
+    def still_owns_foreground_runner() -> bool:
+        from django.db import connections
+
+        try:
+            return command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR)
+        finally:
+            connections.close_all()
+
     def wait_for_turn() -> None:
         raise_if_shutdown_requested()
-        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
         raise_if_shutdown_requested()
 
-    def run_scoped_runner(*args: str) -> None:
+    def run_scoped_runner(*args: str, ensure_daemon_reason: str | None = None) -> None:
         while True:
             wait_for_turn()
-            exit_code = run_runner_worker(list(args), name=f"worker_runner_update_{os.getpid()}")
+            if ensure_daemon_reason:
+                ensure_daemon_stack(reason=ensure_daemon_reason)
+            exit_code = run_runner_worker(
+                list(args),
+                name=f"worker_runner_update_{os.getpid()}",
+                keep_running=still_owns_foreground_runner,
+            )
             if exit_code == 0:
                 return
-            if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+            if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
                 continue
             raise SystemExit(exit_code)
 
@@ -263,8 +284,6 @@ def update(
 
     try:
         wait_for_turn()
-        if stop_daemon_stack:
-            stop_existing_supervisord_process()
 
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             while True:
@@ -272,6 +291,9 @@ def update(
                 do_index = index_only or not migrate_only
                 do_run_until_idle = do_migrate or do_index
                 ran_post_migrate_runner = False
+                full_update_empty = False
+                maintenance_work_queued = False
+                runner_work_queued = False
 
                 if do_migrate:
                     if (
@@ -308,6 +330,8 @@ def update(
                         )
                         print_stats(stats)
                         touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                        maintenance_work_queued = stats.get("queued", 0) > 0
+                        runner_work_queued = runner_work_queued or maintenance_work_queued
                     else:
                         stats_combined = {"phase1": {}, "phase2": {}}
 
@@ -324,6 +348,17 @@ def update(
                             wait_for_turn=wait_for_turn,
                         )
                         print_combined_stats(stats_combined)
+                        full_update_empty = (
+                            stats_combined["phase1"].get("processed", 0) == 0 and stats_combined["phase2"].get("snapshots", 0) == 0
+                        )
+                        maintenance_work_queued = any(
+                            (
+                                stats_combined["phase1"].get("queued", 0),
+                                stats_combined["phase2"].get("queued", 0),
+                                stats_combined["phase2"].get("crawls_sealed", 0),
+                            ),
+                        )
+                        runner_work_queued = runner_work_queued or maintenance_work_queued
 
                     if do_run_until_idle:
                         # Filesystem migration is maintenance on existing
@@ -335,46 +370,108 @@ def update(
                         # rows first, runs the targeted plugins, and may leave
                         # the fs_version maintenance tick hidden behind that
                         # plugin work until another update pass.
-                        print("[*] Phase 3: Running filesystem maintenance until idle...")
-                        if is_filtered_update:
+                        if full_update_empty:
+                            print("[*] No snapshots or legacy archive directories found; skipping filesystem maintenance runner.")
+                        elif not maintenance_work_queued:
+                            print("[*] No filesystem maintenance work queued; skipping filesystem maintenance runner.")
+                        else:
+                            print("[*] Phase 3: Running filesystem maintenance until idle...")
+                        if full_update_empty:
+                            pass
+                        elif not maintenance_work_queued:
+                            pass
+                        elif is_filtered_update:
                             if not touched_snapshot_ids:
                                 print("[*] No matching snapshots queued work for the runner.")
                             for snapshot_id in sorted(touched_snapshot_ids):
                                 run_scoped_runner("--snapshot-id", snapshot_id)
                         else:
-                            run_scoped_runner("--maintenance-only")
+                            run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size))
                         ran_post_migrate_runner = True
 
                 if do_index:
-                    ensure_daemon_stack(reason="search indexing")
-                    search_plugins = _get_search_indexing_plugins()
-                    if not search_plugins:
-                        print("[*] No search indexing plugins are available, nothing to backfill.")
+                    if full_update_empty:
+                        print("[*] No snapshots found; skipping search indexing backfill.")
                     else:
-                        snapshots = _build_filtered_snapshots_queryset(
-                            filter_patterns=filter_patterns,
-                            filter_type=filter_type,
-                            status=status,
-                            url__icontains=url__icontains,
-                            url__istartswith=url__istartswith,
-                            tag=tag,
-                            crawl_id=crawl_id,
-                            limit=limit,
-                            sort=sort,
-                            search=search,
-                            before=before,
-                            after=after,
-                            resume=resume,
-                        )
-                        stats = reindex_snapshots(
-                            snapshots,
-                            search_plugins=search_plugins,
-                            batch_size=batch_size,
-                            collect_ids=is_filtered_update,
-                            wait_for_turn=wait_for_turn,
-                        )
-                        print_index_stats(stats)
-                        touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                        ensure_daemon_stack(reason="search indexing")
+                        search_plugins = _get_search_indexing_plugins()
+                        if not search_plugins:
+                            print("[*] No search indexing plugins are available, nothing to backfill.")
+                        else:
+                            snapshots = _build_filtered_snapshots_queryset(
+                                filter_patterns=filter_patterns,
+                                filter_type=filter_type,
+                                status=status,
+                                url__icontains=url__icontains,
+                                url__istartswith=url__istartswith,
+                                tag=tag,
+                                crawl_id=crawl_id,
+                                limit=limit,
+                                sort=sort,
+                                search=search,
+                                before=before,
+                                after=after,
+                                resume=resume,
+                            )
+                            from django.db.models import Exists, OuterRef, Q
+                            from django.utils import timezone
+                            from archivebox.core.models import ArchiveResult, Snapshot
+
+                            scoped_snapshot_ids = snapshots.order_by().values("id") if is_filtered_update else None
+                            queued_index_results = ArchiveResult.objects.filter(
+                                status=ArchiveResult.StatusChoices.QUEUED,
+                                plugin__in=search_plugins,
+                            )
+                            if scoped_snapshot_ids is not None:
+                                queued_index_results = queued_index_results.filter(snapshot_id__in=scoped_snapshot_ids)
+
+                            if queued_index_results.exists():
+                                runner_work_queued = True
+                                now = timezone.now()
+                                queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
+                                snapshots_to_wake = (
+                                    Snapshot.objects.filter(
+                                        status__in=(Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED),
+                                    )
+                                    .annotate(
+                                        has_queued_index_result=Exists(queued_result_for_snapshot),
+                                    )
+                                    .filter(
+                                        has_queued_index_result=True,
+                                    )
+                                    .filter(
+                                        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+                                    )
+                                )
+                                if scoped_snapshot_ids is not None:
+                                    snapshots_to_wake = snapshots_to_wake.filter(id__in=scoped_snapshot_ids)
+                                woken_count = snapshots_to_wake.update(
+                                    retry_at=now,
+                                    modified_at=now,
+                                )
+                                print(
+                                    "[*] Existing queued search index jobs found; "
+                                    f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.",
+                                )
+                            else:
+                                collect_index_ids = (
+                                    is_filtered_update
+                                    or foreground_runner_owner(
+                                        data_dir=CONSTANTS.DATA_DIR,
+                                        exclude_id=command.id,
+                                    )
+                                    is not None
+                                )
+                                stats = reindex_snapshots(
+                                    snapshots,
+                                    search_plugins=search_plugins,
+                                    batch_size=batch_size,
+                                    collect_ids=collect_index_ids,
+                                    wait_for_turn=wait_for_turn,
+                                )
+                                print_index_stats(stats)
+                                touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                                runner_work_queued = runner_work_queued or stats["queued"] > 0
 
                 if do_run_until_idle and (do_index or not ran_post_migrate_runner):
                     # Search/index backfill intentionally queues targeted
@@ -384,14 +481,26 @@ def update(
                     # For a normal unfiltered `archivebox update`, keep the
                     # historical final pass broad enough to resume genuinely
                     # queued/interrupted crawl work after maintenance is done.
-                    print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
-                    if is_filtered_update:
+                    if full_update_empty:
+                        print("[*] No snapshots found; skipping queued/interrupted crawl runner.")
+                    elif not runner_work_queued:
+                        print("[*] No queued/interrupted crawl work found; skipping queued/interrupted crawl runner.")
+                    else:
+                        print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
+                    if full_update_empty:
+                        pass
+                    elif not runner_work_queued:
+                        pass
+                    elif touched_snapshot_ids and is_filtered_update:
                         if not touched_snapshot_ids:
                             print("[*] No matching snapshots queued work for the runner.")
                         for snapshot_id in sorted(touched_snapshot_ids):
                             run_scoped_runner("--snapshot-id", snapshot_id)
                     else:
-                        run_scoped_runner(*(["--maintenance-only"] if index_only or migrate_only else []))
+                        run_scoped_runner(
+                            *(["--maintenance-only", "--maintenance-batch-size", str(batch_size)] if index_only or migrate_only else []),
+                            ensure_daemon_reason="search indexing" if do_index else None,
+                        )
 
                 if not continuous:
                     break
@@ -401,7 +510,7 @@ def update(
                 resume = None
     except (KeyboardInterrupt, asyncio.CancelledError) as err:
         exit_code = 130
-        exact_resume = getattr(err, "archivebox_resume", None)
+        exact_resume = err.__dict__.get("archivebox_resume")
         resume_cmd = ["archivebox", "update"]
         if migrate_only:
             resume_cmd.append("--migrate-only")
@@ -464,7 +573,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
     1:1 mapping between DB and filesystem.
     """
     from archivebox.core.models import Snapshot
-    from archivebox.config.common import get_config
+    from archivebox.config import CONSTANTS
     from archivebox.crawls.models import Crawl
     from django.utils import timezone
 
@@ -473,8 +582,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
     crawl_url_sets: dict[str, set[str]] = {}
     dirty_crawl_ids: set[str] = set()
 
-    runtime_config = get_config()
-    archive_dir = runtime_config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
     if not archive_dir.exists():
         return stats
 
@@ -570,7 +678,11 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
             # write to one statement while the migration loop does filesystem
             # work outside any transaction. The modified_at CAS prevents this
             # repair scan from overwriting a newer Snapshot edit.
-            if not snapshot.safe_update({"crawl": crawl}, refresh=False):
+            if not snapshot.safe_update(
+                {"crawl": crawl},
+                refresh=False,
+                extra_filter={"modified_at": snapshot.modified_at},
+            ):
                 stats["skipped"] += 1
                 print(f"    [{stats['processed']}] Skipped stale snapshot repair: {entry_path.name}")
                 continue
@@ -579,7 +691,11 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
         # Check if needs migration (0.8.x → 0.9.x)
         try:
             if snapshot.fs_migration_needed:
-                if snapshot.safe_update({"retry_at": timezone.now(), "modified_at": timezone.now()}, refresh=False):
+                if snapshot.safe_update(
+                    {"retry_at": timezone.now(), "modified_at": timezone.now()},
+                    refresh=False,
+                    extra_filter={"modified_at": snapshot.modified_at},
+                ):
                     stats["queued"] += 1
                     print(f"    [{stats['processed']}] Queued filesystem migration: {entry_path.name}")
                 else:
@@ -621,6 +737,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     """
     from archivebox.core.models import Snapshot
     from archivebox.crawls.models import Crawl
+    from django.db.models import Q
     from django.utils import timezone
 
     stats = {
@@ -630,7 +747,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
         "updated_db": 0,
         "queued": 0,
         "sealed": 0,
-        "crawls_queued": 0,
+        "crawls_sealed": 0,
     }
     current_fs_version = Snapshot._fs_current_version()
 
@@ -638,6 +755,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     if resume:
         queryset = queryset.filter(timestamp__lte=resume)
     total = queryset.count()
+    stats["snapshots"] = total
     print(f"[*] Processing {total} snapshots from database (most recent first)...")
 
     def update_in_batches(rows, *, label: str, **updates) -> int:
@@ -659,7 +777,13 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
                 # do filesystem migration work that belongs in the runner.
                 # Guard each single-row UPDATE with modified_at so stale scan
                 # pages cannot overwrite newer runner/admin writes.
-                updated += int(snapshot.safe_update(updates, refresh=False))
+                updated += int(
+                    snapshot.safe_update(
+                        updates,
+                        refresh=False,
+                        extra_filter={"modified_at": snapshot.modified_at},
+                    ),
+                )
             print(f"    [{label}] updated {updated} rows so far")
 
     now = timezone.now()
@@ -680,7 +804,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     stats["sealed"] += updated_rows
     stats["updated_db"] += updated_rows
 
-    fs_version_rows = queryset.exclude(fs_version=current_fs_version)
+    fs_version_rows = queryset.exclude(fs_version=current_fs_version).filter(Q(retry_at__isnull=True) | Q(retry_at__gt=now))
     stale_batch = []
 
     def queue_stale_fs_batch() -> None:
@@ -730,23 +854,23 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     queue_stale_fs_batch()
 
     now = timezone.now()
-    stats["crawls_queued"] = (
+    # Crawls with no open child snapshots are already finished. Seal them here
+    # instead of waking the foreground runner; otherwise migration/update can
+    # accidentally re-enter full crawl execution for historical rows.
+    stats["crawls_sealed"] = (
         Crawl.objects.filter(
-            status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED],
+            status__in=Crawl.RUNNABLE_STATES,
         )
         .exclude(
-            snapshot_set__status__in=[
-                Snapshot.StatusChoices.QUEUED,
-                Snapshot.StatusChoices.STARTED,
-                Snapshot.StatusChoices.PAUSED,
-            ],
+            snapshot_set__status__in=Snapshot.OPEN_STATES,
         )
         .update(
-            retry_at=now,
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
             modified_at=now,
         )
     )
-    stats["updated_db"] += stats["crawls_queued"]
+    stats["updated_db"] += stats["crawls_sealed"]
     return stats
 
 
@@ -826,7 +950,13 @@ def process_filtered_snapshots(
                 # with paged_iterator() and writes later, modified_at is the CAS
                 # guard that prevents stale CLI scans from overwriting a newer
                 # runner/admin update to the same snapshot.
-                updated = int(snapshot.safe_update(update_values, refresh=False))
+                updated = int(
+                    snapshot.safe_update(
+                        update_values,
+                        refresh=False,
+                        extra_filter={"modified_at": snapshot.modified_at},
+                    ),
+                )
                 stats["updated_db"] += updated
 
             stats["queued"] += updated if queue_for_archiving else 0
@@ -878,7 +1008,7 @@ Phase 2 (Process DB):
   Updated JSON:     {s2.get("updated_json", 0)}
   Updated DB rows:  {s2.get("updated_db", 0)}
   Sealed snapshots: {s2.get("sealed", 0)}
-  Queued crawls:    {s2.get("crawls_queued", 0)}
+  Sealed crawls:    {s2.get("crawls_sealed", 0)}
 """)
 
 
@@ -888,28 +1018,19 @@ def print_index_stats(stats: dict[str, Any]) -> None:
     print(f"""
 [green]Search Reindex Complete[/green]
   Scanned rows:      {stats["processed"]}
+  Requested jobs:    {stats.get("requested", stats["queued"])}
   Queued index jobs: {stats["queued"]}
+  Already queued:    {stats.get("skipped_queued", 0)}
 """)
 
 
 @click.command()
 @click.option("--resume", type=str, help="Resume from timestamp")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of snapshots to update")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for positional query")
-@click.option("--before", type=float, help="Only snapshots before timestamp")
-@click.option("--after", type=float, help="Only snapshots after timestamp")
-@click.option("--filter-type", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact")
 @click.option("--batch-size", type=int, default=500, help="Commit every N records")
 @click.option("--continuous", is_flag=True, help="Run continuously as background worker")
 @click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content")
 @click.option("--migrate-only", is_flag=True, help="Only migrate filesystem and update database/index state")
-@click.argument("filter_patterns", nargs=-1)
+@snapshot_filter_options(default_filter_type="exact")
 @docstring(update.__doc__)
 def main(**kwargs):
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
@@ -917,6 +1038,8 @@ def main(**kwargs):
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             update(**kwargs)
+    except ValueError as err:
+        raise click.BadParameter(str(err), param_hint="--status") from err
     except KeyboardInterrupt:
         raise SystemExit(130) from None
 

archivebox/cli/archivebox_version.py

@@ -32,7 +32,6 @@ def _format_binary_abspath(
         (lib_dir, "LIB_DIR/"),
         (Path(os.environ.get("LIB_DIR", "")), "LIB_DIR/") if os.environ.get("LIB_DIR") else (Path(), ""),
         (personas_dir, "PERSONAS_DIR/"),
-        (Path(os.environ.get("PERSONAS_DIR", "")), "PERSONAS_DIR/") if os.environ.get("PERSONAS_DIR") else (Path(), ""),
         (home, "~/"),
     )
 
@@ -134,7 +133,7 @@ def version(
     try:
         DATA_DIR_STAT = CONSTANTS.DATA_DIR.stat()
         prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
             f"FS_UID={DATA_DIR_STAT.st_uid}:{DATA_DIR_STAT.st_gid}",
             f"FS_PERMS={config.OUTPUT_PERMISSIONS}",
             f"FS_ATOMIC={config.ENFORCE_ATOMIC_WRITES}",
@@ -142,7 +141,7 @@ def version(
         )
     except Exception:
         prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
         )
 
     prnt(
@@ -155,7 +154,7 @@ def version(
     )
     prnt()
 
-    if not (os.access(config.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
+    if not (os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
         PANEL_TEXT = "\n".join(
             (
                 "",
@@ -219,7 +218,7 @@ def version(
                             installed.abspath,
                             pwd=Path.cwd(),
                             lib_dir=config.LIB_DIR,
-                            personas_dir=config.PERSONAS_DIR,
+                            personas_dir=CONSTANTS.PERSONAS_DIR,
                             home=Path.home(),
                         )
                         if compact_paths
@@ -275,7 +274,7 @@ def version(
             prnt(f"  [red]Error getting code locations: {e}[/red]")
 
         prnt()
-        if os.access(config.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
+        if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
             prnt("[bright_yellow][i] Data locations:[/bright_yellow]")
             try:
                 for name, path in get_data_locations().items():

archivebox/config/collection.py

@@ -1,36 +1,297 @@
 __package__ = "archivebox.config"
 
+import io
+import json
 import os
+from collections.abc import Mapping
+from typing import Any
 
 from archivebox.config.constants import CONSTANTS
 from archivebox.config.configset import CaseConfigParser
 from archivebox.misc.logging import AttrDict
 
 
-def write_config_file(config: dict[str, str]) -> AttrDict:
-    """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
+CONFIG_FILE_HEADER = (
+    "# This is the config file for your ArchiveBox collection.\n"
+    "#\n"
+    "# You can add options here manually in INI format, or automatically by running:\n"
+    "#    archivebox config --set KEY=VALUE\n"
+    "#\n"
+    "# This file is kept in sync 1:1 with Machine.config in the index DB —\n"
+    "# editing either side propagates to the other. ``archivebox init`` reads\n"
+    "# this file on startup; the admin Machine.config editor writes both.\n"
+    "#\n"
+    "# Full reference:\n"
+    "#    https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration\n"
+    "\n"
+)
 
+
+# Recursion guard for the bidirectional file<->DB mirror. Bidirectional sync
+# means a write on one side always triggers an update on the other; without
+# this flag, ``Machine.save -> mirror to file -> write_config_file -> mirror
+# back to Machine -> Machine.save -> ...`` would loop forever. Module-level
+# scalar is fine here — daphne handles each request on its own thread and the
+# admin/CLI write paths are inherently serialized through the DB.
+_MIRROR_IN_PROGRESS: bool = False
+# One-time-per-process startup sync, gated so subsequent ``Machine.current()``
+# calls collapse to a single boolean check.
+_INITIAL_SYNC_DONE: bool = False
+
+
+def _coerce_to_str_dict(config: Any) -> dict[str, str]:
+    """Project an arbitrary config payload to flat ``{UPPER_KEY: str}`` form.
+
+    INI files only round-trip strings, so we normalize Machine.config values
+    to strings on the way out and accept the same shape on the way back.
+    Pydantic re-coerces types at read time inside ``get_config``.
+
+    Composite values (``dict`` / ``list`` / ``tuple``) are JSON-encoded so
+    they round-trip back through pydantic-settings — ``str(some_dict)``
+    would produce Python's repr (``{'k': 'v'}`` with single quotes), and
+    pydantic-settings refuses to parse that as a ``dict`` field, causing
+    e.g. ``ABX_INSTALL_CACHE`` to crash with ``ValidationError: Input
+    should be a valid dictionary``.
+    """
+    if not config:
+        return {}
+    if not isinstance(config, Mapping):
+        return {}
+    flat: dict[str, str] = {}
+    for key, value in config.items():
+        upper_key = str(key).upper()
+        if value is None:
+            flat[upper_key] = ""
+        elif isinstance(value, (dict, list, tuple)):
+            flat[upper_key] = json.dumps(value, default=str)
+        else:
+            flat[upper_key] = str(value)
+    return flat
+
+
+def _load_file_config_dict() -> tuple[dict[str, str], float | None]:
+    """Return ``(flat_dict, mtime)`` for ``ArchiveBox.conf`` (``({}, None)`` if missing)."""
+    config_path = CONSTANTS.CONFIG_FILE
+    try:
+        mtime = config_path.stat().st_mtime
+    except FileNotFoundError:
+        return {}, None
+    parser = CaseConfigParser()
+    parser.read(config_path)
+    flat = {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+    return flat, mtime
+
+
+def _resolve_section_for_key(key: str, config_sections, plugin_configs) -> str:
+    for section in config_sections.values():
+        if key in type(section).model_fields:
+            return section.toml_section_header
+    for schema in plugin_configs.values():
+        if "properties" in schema and key in schema["properties"]:
+            return "PLUGINS"
+    # Unknown / user-defined keys land in SERVER_CONFIG so we never lose them
+    # (the previous code raised here, which was fine for the CLI-only path but
+    # would break the mirror as soon as anyone added a plugin-tunable that
+    # this process hasn't loaded a schema for).
+    return "SERVER_CONFIG"
+
+
+def _render_config_file_content(config: dict[str, str]) -> str:
+    """Render a flat config dict to INI text, grouped by inferred section."""
     from archivebox.config.common import get_all_configs
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
+
+    config_sections = get_all_configs()
+    plugin_configs = discover_plugin_configs()
+
+    parser = CaseConfigParser()
+    for key, val in sorted(config.items()):
+        section = _resolve_section_for_key(key, config_sections, plugin_configs)
+        if section not in parser:
+            parser[section] = {}
+        parser[section][key] = "" if val is None else str(val)
+
+    buf = io.StringIO()
+    buf.write(CONFIG_FILE_HEADER)
+    parser.write(buf)
+    return buf.getvalue()
+
+
+def _write_file_if_changed(content: str) -> bool:
+    """Atomic-write ``ArchiveBox.conf`` only when contents actually differ.
+
+    Skipping unchanged writes is the difference between every Machine.save in
+    a hot autodetection loop costing one disk write vs. zero.
+    """
     from archivebox.misc.system import atomic_write
 
-    CONFIG_HEADER = """# This is the config file for your ArchiveBox collection.
-    #
-    # You can add options here manually in INI format, or automatically by running:
-    #    archivebox config --set KEY=VALUE
-    #
-    # If you modify this file manually, make sure to update your archive after by running:
-    #    archivebox init
-    #
-    # A list of all possible config with documentation and examples can be found here:
-    #    https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration
+    config_path = CONSTANTS.CONFIG_FILE
+    try:
+        existing = config_path.read_text(encoding="utf-8") if config_path.exists() else None
+    except OSError:
+        existing = None
+    if existing == content:
+        return False
+    atomic_write(config_path, content)
+    return True
 
+
+def mirror_machine_config_to_file(config: Any) -> None:
+    """Rewrite ``ArchiveBox.conf`` so it mirrors ``Machine.config`` exactly.
+
+    Called from ``Machine.save`` after the row is committed. Recursion-guarded
+    so the matching write_config_file -> Machine.save bounce doesn't loop.
     """
+    global _MIRROR_IN_PROGRESS
+    if _MIRROR_IN_PROGRESS:
+        return
+    _MIRROR_IN_PROGRESS = True
+    try:
+        flat = _coerce_to_str_dict(config)
+        _write_file_if_changed(_render_config_file_content(flat))
+    finally:
+        _MIRROR_IN_PROGRESS = False
+
+
+def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
+    """Inverse of ``_coerce_to_str_dict``: decode complex INI values to native.
+
+    ``mirror_machine_config_to_file`` JSON-encodes ``dict`` / ``list`` values
+    so they round-trip through INI's string-only storage. When reading the
+    file back into ``Machine.config`` (a JSONField that holds native types)
+    those strings have to be decoded — otherwise downstream consumers like
+    ``_emit_machine_config`` → ``MachineEvent`` → abx-dl see a JSON string
+    where they expect a dict and raise ``TypeError``.
+    Declared fields go through pydantic-settings' own ``field_is_complex`` /
+    ``prepare_field_value`` so they're decoded per annotation. Undeclared
+    keys (e.g. ``ABX_INSTALL_CACHE``, written dynamically by abx-dl) are
+    JSON-decoded when their string starts with ``{`` or ``[`` — the same
+    shape ``_coerce_to_str_dict`` writes them as.
+    """
+    from archivebox.config.common import ArchiveBoxConfig
+    from archivebox.config.configset import IniConfigSettingsSource
+
+    decoder = IniConfigSettingsSource(ArchiveBoxConfig)
+    decoded: dict[str, Any] = dict(file_config)
+    declared_fields = set(ArchiveBoxConfig.model_fields)
+    for field_name, field in ArchiveBoxConfig.model_fields.items():
+        if field_name not in decoded:
+            continue
+        raw = decoded[field_name]
+        if not isinstance(raw, str) or not raw:
+            continue
+        if decoder.field_is_complex(field):
+            decoded[field_name] = decoder.prepare_field_value(field_name, field, raw, True)
+    for key, raw in list(decoded.items()):
+        if key in declared_fields:
+            continue
+        if not isinstance(raw, str) or not raw:
+            continue
+        first = raw[:1]
+        if first not in ("{", "["):
+            continue
+        try:
+            decoded[key] = json.loads(raw)
+        except (TypeError, ValueError):
+            continue
+    return decoded
+
+
+def _mirror_file_to_machine_config(file_config: dict[str, str]) -> None:
+    """Copy ``ArchiveBox.conf`` contents into ``Machine.config``.
+
+    Internal helper used by ``write_config_file`` and the startup sync —
+    callers must hold the ``_MIRROR_IN_PROGRESS`` guard around it.
+    """
+    from archivebox.machine.models import Machine
+
+    machine = Machine.current()
+    if _coerce_to_str_dict(machine.config) == file_config:
+        return
+    machine.config = _coerce_from_str_dict(file_config)
+    machine.save(update_fields=["config", "modified_at"])
+
+
+def sync_machine_and_file(machine: Any = None) -> None:
+    """One-time-per-process reconciliation between the two stores.
+
+    Cheap on the common case where they already agree (single ``stat`` + dict
+    compare ≈ 1ms). When the two sides diverge we merge them: each side's
+    unique keys are preserved, and for keys present on both we let the newer
+    side win (file mtime vs. ``Machine.modified_at``). After the merge both
+    stores hold the union, so every subsequent write keeps them in lockstep
+    via the full-replace mirror functions.
+
+    Pass ``machine`` when the caller already has a current ``Machine``
+    instance (e.g. from ``Machine.current()``) to skip the 10–15ms
+    ``get_host_guid()`` round-trip on the cold path.
+    """
+    global _INITIAL_SYNC_DONE, _MIRROR_IN_PROGRESS
+    if _INITIAL_SYNC_DONE:
+        return
+    _INITIAL_SYNC_DONE = True
+    if _MIRROR_IN_PROGRESS:
+        return
+    _MIRROR_IN_PROGRESS = True
+    try:
+        if machine is None:
+            from archivebox.machine.detect import get_host_guid
+            from archivebox.machine.models import Machine
+
+            try:
+                machine = Machine.objects.filter(guid=get_host_guid()).first()
+            except Exception:
+                return
+            if machine is None:
+                return
+
+        file_config, file_mtime = _load_file_config_dict()
+        machine_config = _coerce_to_str_dict(machine.config)
+        if machine_config == file_config:
+            return
+
+        db_mtime = machine.modified_at.timestamp() if machine.modified_at else 0.0
+        file_is_newer = file_mtime is not None and file_mtime > db_mtime
+
+        merged: dict[str, str] = {}
+        all_keys = set(machine_config) | set(file_config)
+        for key in all_keys:
+            in_file = key in file_config
+            in_db = key in machine_config
+            if in_file and in_db:
+                if file_config[key] == machine_config[key]:
+                    merged[key] = file_config[key]
+                else:
+                    merged[key] = file_config[key] if file_is_newer else machine_config[key]
+            elif in_file:
+                merged[key] = file_config[key]
+            else:
+                merged[key] = machine_config[key]
+
+        if merged != file_config:
+            _write_file_if_changed(_render_config_file_content(merged))
+        if merged != machine_config:
+            machine.config = _coerce_from_str_dict(merged)
+            machine.save(update_fields=["config", "modified_at"])
+    finally:
+        _MIRROR_IN_PROGRESS = False
+
+
+def write_config_file(config: dict[str, str]) -> AttrDict:
+    """Merge ``config`` into ``ArchiveBox.conf``, validate, then mirror to Machine.config.
+
+    Backwards-compatible signature: callers (CLI ``archivebox config --set``
+    and the init flow) pass a partial dict of keys to upsert.
+    """
+
+    from archivebox.config.common import get_all_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
+    from archivebox.misc.system import atomic_write
 
     config_path = CONSTANTS.CONFIG_FILE
 
     if not os.access(config_path, os.F_OK):
-        atomic_write(config_path, CONFIG_HEADER)
+        atomic_write(config_path, CONFIG_FILE_HEADER)
 
     config_file = CaseConfigParser()
     config_file.read(config_path)
@@ -43,21 +304,7 @@ def write_config_file(config: dict[str, str]) -> AttrDict:
 
     # Set up sections in empty config file
     for key, val in config.items():
-        section_name = None
-        for section in config_sections.values():
-            if key in type(section).model_fields:
-                section_name = section.toml_section_header
-                break
-
-        if section_name is None:
-            for schema in plugin_configs.values():
-                if "properties" in schema and key in schema["properties"]:
-                    section_name = "PLUGINS"
-                    break
-
-        if section_name is None:
-            raise ValueError(f"No config section found for key: {key}")
-
+        section_name = _resolve_section_for_key(key, config_sections, plugin_configs)
         if section_name in config_file:
             existing_config = dict(config_file[section_name])
         else:
@@ -84,4 +331,18 @@ def write_config_file(config: dict[str, str]) -> AttrDict:
     if os.access(f"{config_path}.bak", os.F_OK):
         os.remove(f"{config_path}.bak")
 
+    # Mirror the post-write file state into Machine.config so the DB stays
+    # 1:1 with the on-disk file. Recursion-guarded so Machine.save's own
+    # mirror-back doesn't loop us.
+    global _MIRROR_IN_PROGRESS
+    if not _MIRROR_IN_PROGRESS:
+        _MIRROR_IN_PROGRESS = True
+        try:
+            flat, _mtime = _load_file_config_dict()
+            _mirror_file_to_machine_config(flat)
+        except Exception:
+            pass
+        finally:
+            _MIRROR_IN_PROGRESS = False
+
     return AttrDict({key.upper(): updated_config.get(key.upper()) for key in config.keys()})

archivebox/config/configset.py

@@ -22,8 +22,6 @@ COMPUTED_CONFIG_KEYS = (
     "IS_LOWER_SECURITY_MODE",
     "URL_ALLOWLIST_PTN",
     "URL_DENYLIST_PTN",
-    "SAVE_ALLOWLIST_PTNS",
-    "SAVE_DENYLIST_PTNS",
 )
 
 
@@ -32,6 +30,33 @@ class CaseConfigParser(ConfigParser):
         return optionstr
 
 
+# ``IniConfigSettingsSource.get_field_value`` is called once per pydantic field
+# (hundreds of fields across the GlobalConfig + plugin configs). Without
+# caching, every field re-opens and re-parses ``ArchiveBox.conf`` from disk —
+# that's the dominant cost of ``get_config()`` (each call was ~130ms; profile
+# showed 632 ``parser.read(config_path)`` invocations per call). The cache is
+# keyed on (path, mtime) so external edits to the file still get picked up.
+_INI_CACHE: dict[tuple[str, float], dict[str, Any]] = {}
+
+
+def _read_ini_config_cached(config_path_str: str) -> dict[str, Any]:
+    config_path = Path(config_path_str)
+    try:
+        mtime = config_path.stat().st_mtime
+    except FileNotFoundError:
+        return {}
+    cache_key = (str(config_path), mtime)
+    cached = _INI_CACHE.get(cache_key)
+    if cached is not None:
+        return cached
+    parser = CaseConfigParser()
+    parser.read(config_path)
+    flat = {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+    _INI_CACHE.clear()
+    _INI_CACHE[cache_key] = flat
+    return flat
+
+
 class IniConfigSettingsSource(PydanticBaseSettingsSource):
     """
     Custom settings source that reads from ArchiveBox.conf (INI format).
@@ -41,10 +66,27 @@ class IniConfigSettingsSource(PydanticBaseSettingsSource):
     def get_field_value(self, field: Any, field_name: str) -> tuple[Any, str, bool]:
         config_vals = self._load_config_file()
         field_value = config_vals.get(field_name.upper())
-        return field_value, field_name, False
+        # Mark complex-typed fields (``dict``/``list``) so the parent
+        # ``prepare_field_value`` JSON-decodes the INI string before pydantic
+        # validates against the dict/list annotation. Without this, e.g.
+        # ``ABX_INSTALL_CACHE`` arrives as a raw JSON string and pydantic
+        # rejects it with ``Input should be a valid dictionary``.
+        value_is_complex = bool(field_value is not None and self.field_is_complex(field))
+        return field_value, field_name, value_is_complex
 
     def __call__(self) -> dict[str, Any]:
-        return self._load_config_file()
+        # Use the per-field path (``get_field_value`` + ``prepare_field_value``)
+        # so complex types get JSON-decoded. The previous flat-dict return
+        # skipped pydantic-settings' complex-value handling entirely.
+        result: dict[str, Any] = {}
+        for field_name, field in self.settings_cls.model_fields.items():
+            value, key, value_is_complex = self.get_field_value(field, field_name)
+            if value is None:
+                continue
+            prepared = self.prepare_field_value(field_name, field, value, value_is_complex)
+            if prepared is not None:
+                result[key] = prepared
+        return result
 
     def _load_config_file(self) -> dict[str, Any]:
         try:
@@ -54,14 +96,7 @@ class IniConfigSettingsSource(PydanticBaseSettingsSource):
         except ImportError:
             return {}
 
-        if not config_path.exists():
-            return {}
-
-        parser = CaseConfigParser()
-        parser.read(config_path)
-
-        # Flatten all sections into single namespace (ignore section headers)
-        return {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+        return _read_ini_config_cached(str(config_path))
 
 
 class BaseConfigSet(BaseSettings):

archivebox/config/constants.py

@@ -36,8 +36,8 @@ from .permissions import (
     IN_DOCKER,
     RUNNING_AS_UID,
     RUNNING_AS_GID,
-    DEFAULT_PUID,
-    DEFAULT_PGID,
+    DEFAULT_UID,
+    DEFAULT_GID,
     ARCHIVEBOX_USER,
     ARCHIVEBOX_GROUP,
 )
@@ -66,8 +66,8 @@ class ConstantsDict:
     ARCHIVEBOX_GROUP: int = ARCHIVEBOX_GROUP
     RUNNING_AS_UID: int = RUNNING_AS_UID
     RUNNING_AS_GID: int = RUNNING_AS_GID
-    DEFAULT_PUID: int = DEFAULT_PUID
-    DEFAULT_PGID: int = DEFAULT_PGID
+    DEFAULT_UID: int = DEFAULT_UID
+    DEFAULT_GID: int = DEFAULT_GID
     IS_INSIDE_VENV: bool = sys.prefix != sys.base_prefix
 
     # Source code dirs
@@ -118,7 +118,6 @@ class ConstantsDict:
         "LIB_DIR",
         user_config_path("abx") / LIB_DIR_NAME,
     )
-    DEFAULT_LIB_BIN_DIR: Path = _env_path("LIB_BIN_DIR", DEFAULT_LIB_DIR / "bin")
 
     RESERVED_ARCHIVE_DIR_NAMES: frozenset[str] = frozenset(
         (

archivebox/config/ldap.py

@@ -1,6 +1,6 @@
 __package__ = "archivebox.config"
 
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 
 from archivebox.config.configset import BaseConfigSet
 
@@ -14,6 +14,7 @@ class LDAPConfig(BaseConfigSet):
     """
 
     toml_section_header: str = "LDAP_CONFIG"
+    _scope: str = PrivateAttr(default="server")
 
     LDAP_ENABLED: bool = Field(default=False)
     LDAP_SERVER_URI: str | None = Field(default=None)

archivebox/config/paths.py

@@ -19,7 +19,9 @@ if TYPE_CHECKING:
 #############################################################################################
 
 PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent  # archivebox source code dir
-DATA_DIR: Path = Path(os.environ.get("DATA_DIR", os.getcwd())).resolve()  # archivebox user data dir
+DATA_DIR: Path = Path(os.getcwd()).resolve()  # archivebox user data dir
+MAX_TMP_SOCKET_URL_LENGTH = 90
+SUPERVISORD_SOCKET_FILENAME = "supervisord.sock"
 
 
 def _env_path(key: str, default: Path) -> Path:
@@ -29,8 +31,8 @@ def _env_path(key: str, default: Path) -> Path:
     return path.resolve()
 
 
-ARCHIVE_DIR: Path = _env_path("ARCHIVE_DIR", DATA_DIR / "archive")  # archivebox snapshot data dir
-USERS_DIR: Path = _env_path("USERS_DIR", ARCHIVE_DIR / "users")  # archivebox user-scoped crawl/snapshot data dir
+ARCHIVE_DIR: Path = DATA_DIR / "archive"  # archivebox snapshot data dir
+USERS_DIR: Path = ARCHIVE_DIR / "users"  # archivebox user-scoped crawl/snapshot data dir
 
 IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
 
@@ -120,7 +122,7 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No
     try:
         with SudoPermission(uid=uid, fallback=fallback):
             test_file.exists()
-            test_file.write_text(f"Checking if PUID={uid} PGID={gid} can write to dir")
+            test_file.write_text(f"Checking if uid={uid} gid={gid} can write to dir")
             test_file.unlink()
             return True
     except (OSError, PermissionError):
@@ -176,33 +178,45 @@ def create_and_chown_dir(dir_path: Path) -> None:
 
 
 def tmp_dir_socket_path_is_short_enough(dir_path: Path) -> bool:
-    socket_file = dir_path.absolute().resolve() / "supervisord.sock"
-    return len(f"file://{socket_file}") <= 96
+    socket_file = dir_path.absolute().resolve() / SUPERVISORD_SOCKET_FILENAME
+    return len(f"file://{socket_file}") < MAX_TMP_SOCKET_URL_LENGTH
+
+
+def tmp_dir_candidates(config: "ArchiveBoxConfig") -> list[Path]:
+    from archivebox.config.constants import CONSTANTS
+
+    collection_id = get_collection_id()
+    collection_id_short = collection_id[:4]
+    system_tmp_dir = Path(tempfile.gettempdir())
+    candidates = [
+        config.TMP_DIR,  # <user-specified>
+        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
+        Path("/var/run/archivebox") / collection_id,
+        Path("/tmp") / "archivebox" / collection_id,
+        Path("~/.tmp/archivebox").expanduser() / collection_id,
+        system_tmp_dir / "archivebox" / collection_id,
+        system_tmp_dir / "archivebox" / collection_id_short,
+        system_tmp_dir / "abx" / collection_id_short,
+    ]
+    seen = set()
+    unique_candidates = []
+    for path in candidates:
+        path_key = str(path.expanduser().absolute())
+        if path_key in seen:
+            continue
+        seen.add(path_key)
+        unique_candidates.append(path)
+    return unique_candidates
 
 
 def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
-    from archivebox.config.constants import CONSTANTS
     from archivebox.config.common import get_config
     from archivebox.misc.checks import check_tmp_dir
 
     config = config or get_config(**config_kwargs)
-    # try a few potential directories in order of preference
-    CANDIDATES = [
-        config.TMP_DIR,  # <user-specified>
-        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
-        Path("/var/run/archivebox") / get_collection_id(),  # /var/run/archivebox/abc5d8512
-        Path("/tmp") / "archivebox" / get_collection_id(),  # /tmp/archivebox/abc5d8512
-        Path("~/.tmp/archivebox").expanduser() / get_collection_id(),  # ~/.tmp/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id(),  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d
-        Path(tempfile.gettempdir()) / "abx" / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/abx/abc5
-    ]
+    candidates = tmp_dir_candidates(config)
     fallback_candidate = None
-    for candidate in CANDIDATES:
+    for candidate in candidates:
         try:
             create_and_chown_dir(candidate)
         except Exception:
@@ -231,7 +245,7 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxC
         return fallback_candidate
 
     if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {CANDIDATES}!")
+        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {candidates}!")
 
 
 def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@@ -290,20 +304,20 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
             },
             "ARCHIVE_DIR": {
-                "path": config.ARCHIVE_DIR.resolve(),
+                "path": CONSTANTS.ARCHIVE_DIR.resolve(),
                 "enabled": True,
-                "is_valid": os.path.isdir(config.ARCHIVE_DIR)
-                and os.access(config.ARCHIVE_DIR, os.R_OK)
-                and os.access(config.ARCHIVE_DIR, os.W_OK),
-                "is_mount": os.path.ismount(config.ARCHIVE_DIR.resolve()),
+                "is_valid": os.path.isdir(CONSTANTS.ARCHIVE_DIR)
+                and os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK)
+                and os.access(CONSTANTS.ARCHIVE_DIR, os.W_OK),
+                "is_mount": os.path.ismount(CONSTANTS.ARCHIVE_DIR.resolve()),
             },
             "USERS_DIR": {
-                "path": config.USERS_DIR.resolve(),
-                "enabled": os.path.isdir(config.USERS_DIR),
-                "is_valid": os.path.isdir(config.USERS_DIR)
-                and os.access(config.USERS_DIR, os.R_OK)
-                and os.access(config.USERS_DIR, os.W_OK),
-                "is_mount": os.path.ismount(config.USERS_DIR.resolve()),
+                "path": CONSTANTS.USERS_DIR.resolve(),
+                "enabled": os.path.isdir(CONSTANTS.USERS_DIR),
+                "is_valid": os.path.isdir(CONSTANTS.USERS_DIR)
+                and os.access(CONSTANTS.USERS_DIR, os.R_OK)
+                and os.access(CONSTANTS.USERS_DIR, os.W_OK),
+                "is_mount": os.path.ismount(CONSTANTS.USERS_DIR.resolve()),
             },
             "SOURCES_DIR": {
                 "path": CONSTANTS.SOURCES_DIR.resolve(),
@@ -329,7 +343,10 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
             "TMP_DIR": {
                 "path": tmp_dir.resolve(),
                 "enabled": True,
-                "is_valid": os.path.isdir(tmp_dir) and os.access(tmp_dir, os.R_OK) and os.access(tmp_dir, os.W_OK),  # read + write
+                "is_valid": os.path.isdir(tmp_dir)
+                and os.access(tmp_dir, os.R_OK)
+                and os.access(tmp_dir, os.W_OK)
+                and tmp_dir_socket_path_is_short_enough(tmp_dir),
             },
             # "CACHE_DIR": {
             #     "path": CACHE_DIR.resolve(),
@@ -351,8 +368,6 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
     except Exception:
         lib_dir = config.LIB_DIR
 
-    lib_bin_dir = lib_dir / "bin"
-
     return AttrDict(
         {
             "PACKAGE_DIR": {
@@ -366,9 +381,9 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 "is_valid": os.access(CONSTANTS.STATIC_DIR, os.R_OK) and os.access(CONSTANTS.STATIC_DIR, os.X_OK),  # read + list
             },
             "CUSTOM_TEMPLATES_DIR": {
-                "path": config.CUSTOM_TEMPLATES_DIR.resolve(),
-                "enabled": os.path.isdir(config.CUSTOM_TEMPLATES_DIR),
-                "is_valid": os.path.isdir(config.CUSTOM_TEMPLATES_DIR) and os.access(config.CUSTOM_TEMPLATES_DIR, os.R_OK),  # read
+                "path": CONSTANTS.CUSTOM_TEMPLATES_DIR.resolve(),
+                "enabled": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR),
+                "is_valid": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK),  # read
             },
             "USER_PLUGINS_DIR": {
                 "path": CONSTANTS.USER_PLUGINS_DIR.resolve(),
@@ -380,12 +395,5 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 "enabled": True,
                 "is_valid": os.path.isdir(lib_dir) and os.access(lib_dir, os.R_OK) and os.access(lib_dir, os.W_OK),  # read + write
             },
-            "LIB_BIN_DIR": {
-                "path": lib_bin_dir.resolve(),
-                "enabled": True,
-                "is_valid": os.path.isdir(lib_bin_dir)
-                and os.access(lib_bin_dir, os.R_OK)
-                and os.access(lib_bin_dir, os.W_OK),  # read + write
-            },
         },
     )

archivebox/config/permissions.py

@@ -24,8 +24,8 @@ except PermissionError:
     DATA_DIR_UID = 0
     DATA_DIR_GID = 0
 
-DEFAULT_PUID = 911
-DEFAULT_PGID = 911
+DEFAULT_UID = 911
+DEFAULT_GID = 911
 RUNNING_AS_UID = os.getuid()
 RUNNING_AS_GID = os.getgid()
 EUID = os.geteuid()
@@ -42,23 +42,20 @@ IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE",
 
 FALLBACK_UID = RUNNING_AS_UID or SUDO_UID
 FALLBACK_GID = RUNNING_AS_GID or SUDO_GID
-if RUNNING_AS_UID == 0:
-    try:
-        # if we are running as root it's really hard to figure out what the correct archivebox user should be
-        # as a last resort instead of setting DATA_DIR ownership to 0:0 (which breaks it for non-root users)
-        # check if 911:911 archivebox user exists on host system, and use it instead of 0
-        if pwd.getpwuid(DEFAULT_PUID).pw_name == "archivebox":
-            FALLBACK_UID = DEFAULT_PUID
-            FALLBACK_GID = DEFAULT_PGID
-    except Exception:
-        pass
+try:
+    ARCHIVEBOX_ACCOUNT = pwd.getpwnam("archivebox")
+except KeyError:
+    ARCHIVEBOX_ACCOUNT = None
 
-
-os.environ.setdefault("PUID", str(DATA_DIR_UID or EUID or RUNNING_AS_UID or FALLBACK_UID))
-os.environ.setdefault("PGID", str(DATA_DIR_GID or EGID or RUNNING_AS_GID or FALLBACK_GID))
-
-ARCHIVEBOX_USER = int(os.environ["PUID"])
-ARCHIVEBOX_GROUP = int(os.environ["PGID"])
+if DATA_DIR_UID != 0:
+    ARCHIVEBOX_USER = DATA_DIR_UID
+    ARCHIVEBOX_GROUP = DATA_DIR_GID
+elif RUNNING_AS_UID == 0 and ARCHIVEBOX_ACCOUNT is not None:
+    ARCHIVEBOX_USER = ARCHIVEBOX_ACCOUNT.pw_uid
+    ARCHIVEBOX_GROUP = ARCHIVEBOX_ACCOUNT.pw_gid
+else:
+    ARCHIVEBOX_USER = EUID or RUNNING_AS_UID or FALLBACK_UID
+    ARCHIVEBOX_GROUP = EGID or RUNNING_AS_GID or FALLBACK_GID
 if not USER:
     try:
         # alternative method 1 to get username
@@ -94,13 +91,12 @@ except Exception:
 
 
 def drop_privileges():
-    """If running as root, drop privileges to the user that owns the data dir (or PUID)"""
+    """If running as root, drop privileges to the data dir owner or archivebox user."""
 
-    # always run archivebox as the user that owns the data dir, never as root
+    # Always run ArchiveBox as the user that owns the data dir, or as the
+    # archivebox service account when the data dir is root-owned.
     if os.getuid() == 0:
-        # drop permissions to the user that owns the data dir / provided PUID
         if os.geteuid() != ARCHIVEBOX_USER and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
-            # drop our effective UID to the archivebox user's UID
             os.seteuid(ARCHIVEBOX_USER)
 
             # update environment variables so that subprocesses dont try to write to /root

archivebox/config/version.py

@@ -6,6 +6,7 @@ import importlib.metadata
 from pathlib import Path
 from functools import cache
 from datetime import datetime
+import re
 
 #############################################################################################
 
@@ -41,11 +42,57 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR):
 
 @cache
 def get_COMMIT_HASH() -> str | None:
+    for env_var in ("ARCHIVEBOX_COMMIT_HASH", "COMMIT_HASH"):
+        env_commit_hash = os.environ.get(env_var, "").strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", env_commit_hash):
+            return env_commit_hash
+
+    if IN_DOCKER:
+        try:
+            version_txt = Path("/VERSION.txt").read_text()
+            docker_commit_hashes = re.findall(r"COMMIT_HASH=([0-9a-fA-F]{40})", version_txt)
+            if docker_commit_hashes:
+                return docker_commit_hashes[-1]
+        except Exception:
+            pass
+
+    def _read_git_file(git_dir: Path, ref: str) -> str | None:
+        try:
+            return git_dir.joinpath(ref).read_text().strip()
+        except Exception:
+            pass
+
+        try:
+            packed_refs = git_dir.joinpath("packed-refs").read_text().splitlines()
+        except Exception:
+            return None
+
+        for line in packed_refs:
+            if line.startswith("#") or line.startswith("^") or not line.strip():
+                continue
+            commit_hash, packed_ref = line.split(" ", 1)
+            if packed_ref == ref:
+                return commit_hash.strip()
+
+        return None
+
     try:
         git_dir = PACKAGE_DIR.parent / ".git"
-        ref = (git_dir / "HEAD").read_text().strip().split(" ")[-1]
-        commit_hash = git_dir.joinpath(ref).read_text().strip()
-        return commit_hash
+        if git_dir.is_file():
+            gitdir_line = git_dir.read_text().strip()
+            gitdir_path = gitdir_line.removeprefix("gitdir:").strip()
+            git_dir = Path(gitdir_path)
+            if not git_dir.is_absolute():
+                git_dir = PACKAGE_DIR.parent / git_dir
+
+        head = (git_dir / "HEAD").read_text().strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", head):
+            return head
+
+        ref = head.removeprefix("ref:").strip()
+        commit_hash = _read_git_file(git_dir, ref)
+        if commit_hash:
+            return commit_hash
     except Exception:
         pass
 

archivebox/config/views.py

@@ -1,13 +1,9 @@
 __package__ = "archivebox.config"
 
-import html
-import json
 import os
 import inspect
-import re
 from pathlib import Path
 from typing import Any
-from collections.abc import Callable
 from urllib.parse import quote, urlencode
 from django.http import HttpRequest
 from django.utils import timezone
@@ -22,15 +18,12 @@ from archivebox.misc.util import parse_date
 
 from archivebox.machine.models import Binary
 
-ABX_PLUGINS_DOCS_BASE_URL = "https://archivebox.github.io/abx-plugins/"
-ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/"
-LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/"
 INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"
 
 
 def is_superuser(request: HttpRequest) -> bool:
-    return bool(getattr(request.user, "is_superuser", False))
+    return bool(request.user.is_superuser)
 
 
 def format_parsed_datetime(value: object) -> str:
@@ -38,59 +31,6 @@ def format_parsed_datetime(value: object) -> str:
     return parsed.strftime("%Y-%m-%d %H:%M:%S") if parsed else ""
 
 
-JSON_TOKEN_RE = re.compile(
-    r'(?P<key>"(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*")(?=\s*:)'
-    r'|(?P<string>"(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*")'
-    r"|(?P<boolean>\btrue\b|\bfalse\b)"
-    r"|(?P<null>\bnull\b)"
-    r"|(?P<number>-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)",
-)
-
-
-def render_code_block(text: str, *, highlighted: bool = False) -> str:
-    code = html.escape(text, quote=False)
-
-    if highlighted:
-
-        def _wrap_token(match: re.Match[str]) -> str:
-            styles = {
-                "key": "color: #0550ae;",
-                "string": "color: #0a7f45;",
-                "boolean": "color: #8250df; font-weight: 600;",
-                "null": "color: #6e7781; font-style: italic;",
-                "number": "color: #b35900;",
-            }
-            token_type = next(name for name, value in match.groupdict().items() if value is not None)
-            return f'<span style="{styles[token_type]}">{match.group(0)}</span>'
-
-        code = JSON_TOKEN_RE.sub(_wrap_token, code)
-
-    return (
-        '<pre style="max-height: 600px; overflow: auto; background: #f6f8fa; '
-        'border: 1px solid #d0d7de; border-radius: 6px; padding: 12px; margin: 0;">'
-        '<code style="font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, '
-        "'Liberation Mono', monospace; white-space: pre; line-height: 1.5;\">"
-        f"{code}"
-        "</code></pre>"
-    )
-
-
-def render_highlighted_json_block(value: Any) -> str:
-    return render_code_block(json.dumps(value, indent=2, ensure_ascii=False), highlighted=True)
-
-
-def get_plugin_docs_url(plugin_name: str) -> str:
-    return f"{ABX_PLUGINS_DOCS_BASE_URL}#{plugin_name}"
-
-
-def get_plugin_hook_source_url(plugin_name: str, hook_name: str) -> str:
-    return f"{ABX_PLUGINS_GITHUB_BASE_URL}{quote(plugin_name)}/{quote(hook_name)}"
-
-
-def get_live_config_url(key: str) -> str:
-    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
-
-
 def get_environment_binary_url(name: str) -> str:
     return f"{ENVIRONMENT_BINARIES_BASE_URL}{quote(name)}/"
 
@@ -99,200 +39,11 @@ def get_installed_binary_change_url(name: str, binary: Binary | None) -> str | N
     if binary is None or not binary.id:
         return None
 
-    base_url = getattr(binary, "admin_change_url", None) or f"{INSTALLED_BINARIES_BASE_URL}{binary.id}/change/"
+    base_url = binary.admin_change_url
     changelist_filters = urlencode({"q": name})
     return f"{base_url}?{urlencode({'_changelist_filters': changelist_filters})}"
 
 
-def get_machine_admin_url() -> str | None:
-    try:
-        from archivebox.machine.models import Machine
-
-        machine = Machine.current()
-        return getattr(machine, "admin_change_url", None) or f"/admin/machine/machine/{machine.id}/change/"
-    except Exception:
-        return None
-
-
-def render_code_tag_list(values: list[str]) -> str:
-    if not values:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    tags = "".join(
-        str(
-            format_html(
-                '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>',
-                value,
-            ),
-        )
-        for value in values
-    )
-    return f'<div style="display: flex; flex-wrap: wrap;">{tags}</div>'
-
-
-def render_plugin_metadata_html(config: dict[str, Any]) -> str:
-    required_binaries = [
-        str(item.get("name")) for item in (config.get("required_binaries") or []) if isinstance(item, dict) and item.get("name")
-    ]
-    rows = (
-        ("Title", config.get("title") or "(none)"),
-        ("Description", config.get("description") or "(none)"),
-        ("Required Plugins", mark_safe(render_link_tag_list(config.get("required_plugins") or [], get_plugin_docs_url))),
-        ("Required Binaries", mark_safe(render_link_tag_list(required_binaries, get_environment_binary_url))),
-        ("Output MIME Types", mark_safe(render_code_tag_list(config.get("output_mimetypes") or []))),
-    )
-
-    rendered_rows = "".join(
-        str(
-            format_html(
-                '<div style="margin: 0 0 14px 0;"><div style="font-weight: 600; margin-bottom: 4px;">{}</div><div>{}</div></div>',
-                label,
-                value,
-            ),
-        )
-        for label, value in rows
-    )
-    return f'<div style="margin: 4px 0 0 0;">{rendered_rows}</div>'
-
-
-def render_link_tag_list(values: list[str], url_resolver: Callable[[str], str] | None = None) -> str:
-    if not values:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    tags = []
-    for value in values:
-        if url_resolver is None:
-            tags.append(
-                str(
-                    format_html(
-                        '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                        'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>',
-                        value,
-                    ),
-                ),
-            )
-        else:
-            tags.append(
-                str(
-                    format_html(
-                        '<a href="{}" style="text-decoration: none;">'
-                        '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                        'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>'
-                        "</a>",
-                        url_resolver(value),
-                        value,
-                    ),
-                ),
-            )
-    return f'<div style="display: flex; flex-wrap: wrap;">{"".join(tags)}</div>'
-
-
-def render_property_links(prop_name: str, prop_info: dict[str, Any], machine_admin_url: str | None) -> str:
-    links = [
-        str(format_html('<a href="{}">Computed value</a>', get_live_config_url(prop_name))),
-    ]
-    if machine_admin_url:
-        links.append(str(format_html('<a href="{}">Edit override</a>', machine_admin_url)))
-
-    fallback = prop_info.get("x-fallback")
-    if isinstance(fallback, str) and fallback:
-        links.append(str(format_html('<a href="{}">Fallback: <code>{}</code></a>', get_live_config_url(fallback), fallback)))
-
-    aliases = prop_info.get("x-aliases") or []
-    if isinstance(aliases, list):
-        for alias in aliases:
-            if isinstance(alias, str) and alias:
-                links.append(str(format_html('<a href="{}">Alias: <code>{}</code></a>', get_live_config_url(alias), alias)))
-
-    default = prop_info.get("default")
-    if prop_name.endswith("_BINARY") and isinstance(default, str) and default:
-        links.append(str(format_html('<a href="{}">Binary: <code>{}</code></a>', get_environment_binary_url(default), default)))
-
-    return " &nbsp; ".join(links)
-
-
-def render_config_properties_html(properties: dict[str, Any], machine_admin_url: str | None) -> str:
-    header_links = [
-        str(format_html('<a href="{}">Dependencies</a>', ENVIRONMENT_BINARIES_BASE_URL)),
-        str(format_html('<a href="{}">Installed Binaries</a>', INSTALLED_BINARIES_BASE_URL)),
-    ]
-    if machine_admin_url:
-        header_links.insert(0, str(format_html('<a href="{}">Machine Config Editor</a>', machine_admin_url)))
-
-    cards = [
-        f'<div style="margin: 0 0 16px 0;">{" &nbsp; | &nbsp; ".join(header_links)}</div>',
-    ]
-
-    for prop_name, prop_info in properties.items():
-        prop_type = prop_info.get("type", "unknown")
-        if isinstance(prop_type, list):
-            prop_type = " | ".join(str(type_name) for type_name in prop_type)
-        prop_desc = prop_info.get("description", "")
-
-        default_html = ""
-        if "default" in prop_info:
-            default_html = str(
-                format_html(
-                    '<div style="margin-top: 6px;"><b>Default:</b> <code>{}</code></div>',
-                    prop_info["default"],
-                ),
-            )
-
-        description_html = prop_desc or mark_safe('<span style="color: #6e7781;">(no description)</span>')
-        cards.append(
-            str(
-                format_html(
-                    '<div style="margin: 0 0 14px 0; padding: 12px; background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 6px;">'
-                    '<div style="margin-bottom: 6px;">'
-                    '<a href="{}" style="font-weight: 600;"><code>{}</code></a>'
-                    ' <span style="color: #6e7781;">({})</span>'
-                    "</div>"
-                    '<div style="margin-bottom: 6px;">{}</div>'
-                    '<div style="font-size: 0.95em;">{}</div>'
-                    "{}"
-                    "</div>",
-                    get_live_config_url(prop_name),
-                    prop_name,
-                    prop_type,
-                    description_html,
-                    mark_safe(render_property_links(prop_name, prop_info, machine_admin_url)),
-                    mark_safe(default_html),
-                ),
-            ),
-        )
-
-    return "".join(cards)
-
-
-def render_hook_links_html(plugin_name: str, hooks: list[str], source: str) -> str:
-    if not hooks:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    items = []
-    for hook_name in hooks:
-        if source == "builtin":
-            items.append(
-                str(
-                    format_html(
-                        '<div style="margin: 0 0 8px 0;"><a href="{}" target="_blank" rel="noopener noreferrer"><code>{}</code></a></div>',
-                        get_plugin_hook_source_url(plugin_name, hook_name),
-                        hook_name,
-                    ),
-                ),
-            )
-        else:
-            items.append(
-                str(
-                    format_html(
-                        '<div style="margin: 0 0 8px 0;"><code>{}</code></div>',
-                        hook_name,
-                    ),
-                ),
-            )
-    return "".join(items)
-
-
 def render_binary_detail_description(name: str, merged: dict[str, Any], db_binary: Any) -> str:
     installed_binary_url = get_installed_binary_change_url(name, db_binary)
 
@@ -360,59 +111,32 @@ def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]:
 
 
 def get_db_binaries_by_name() -> dict[str, Binary]:
+    """Group Binary rows by a URL-safe canonical name.
+
+    Hooks occasionally emit ``BinaryEvent.name`` carrying an abspath rather
+    than a short binary name (see ``services/binary_service.py``). That used
+    to leak ``name='/Users/.../bin/foo'`` rows into the DB, which then broke
+    ``/admin/environment/binaries`` because the admin URL regex is
+    ``(?P<key>[^/]+)``. Canonicalize at the keying step so duplicates fold
+    into the real binary and the admin link key stays slash-free regardless
+    of legacy DB state.
+    """
+    from archivebox.machine.models import _canonical_binary_name
+
     grouped: dict[str, list[Binary]] = {}
     binary_name_aliases = {
         "youtube-dl": "yt-dlp",
     }
     for binary in Binary.objects.all():
-        canonical_name = binary_name_aliases.get(binary.name, binary.name)
+        canonical_name = _canonical_binary_name(binary.name)
+        canonical_name = binary_name_aliases.get(canonical_name, canonical_name)
+        if not canonical_name:
+            continue
         grouped.setdefault(canonical_name, []).append(binary)
 
     return {name: max(records, key=_binary_sort_key) for name, records in grouped.items()}
 
 
-def get_filesystem_plugins() -> dict[str, dict[str, Any]]:
-    """Discover plugins from filesystem directories."""
-    import json
-    from archivebox.hooks import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR
-
-    plugins = {}
-
-    for base_dir, source in [(BUILTIN_PLUGINS_DIR, "builtin"), (USER_PLUGINS_DIR, "user")]:
-        if not base_dir.exists():
-            continue
-
-        for plugin_dir in base_dir.iterdir():
-            if plugin_dir.is_dir() and not plugin_dir.name.startswith("_"):
-                plugin_id = f"{source}.{plugin_dir.name}"
-
-                # Find hook scripts
-                hooks = []
-                for ext in ("sh", "py", "js"):
-                    hooks.extend(plugin_dir.glob(f"on_*__*.{ext}"))
-
-                # Load config.json if it exists
-                config_file = plugin_dir / "config.json"
-                config_data = None
-                if config_file.exists():
-                    try:
-                        with open(config_file) as f:
-                            config_data = json.load(f)
-                    except (json.JSONDecodeError, OSError):
-                        config_data = None
-
-                plugins[plugin_id] = {
-                    "id": plugin_id,
-                    "name": plugin_dir.name,
-                    "path": str(plugin_dir),
-                    "source": source,
-                    "hooks": [str(h.name) for h in hooks],
-                    "config": config_data,
-                }
-
-    return plugins
-
-
 @render_with_table_view
 def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."
@@ -429,7 +153,7 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and getattr(binary, "is_valid", getattr(binary, "abspath", None)))
+        binary_is_valid = bool(binary and binary.is_valid)
 
         rows["Binary Name"].append(ItemLink(name, key=name))
 
@@ -456,9 +180,9 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
         "youtube-dl": "yt-dlp",
     }.get(key, key)
     db_binary = get_db_binaries_by_name().get(key)
-    binary_is_valid = bool(db_binary and getattr(db_binary, "is_valid", getattr(db_binary, "abspath", None)))
+    binary_is_valid = bool(db_binary and db_binary.is_valid)
     if binary_is_valid:
-        binary_data = db_binary.to_json() if hasattr(db_binary, "to_json") else db_binary.__dict__
+        binary_data = db_binary.to_json()
         section: SectionData = {
             "name": key,
             "description": mark_safe(render_binary_detail_description(key, binary_data, db_binary)),
@@ -497,137 +221,13 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     )
 
 
-@render_with_table_view
-def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    assert is_superuser(request), "Must be a superuser to view configuration settings."
-
-    rows = {
-        "Name": [],
-        "Source": [],
-        "Path": [],
-        "Hooks": [],
-        "Config": [],
-    }
-
-    plugins = get_filesystem_plugins()
-
-    for plugin_id, plugin in plugins.items():
-        rows["Name"].append(ItemLink(plugin["name"], key=plugin_id))
-        rows["Source"].append(plugin["source"])
-        rows["Path"].append(format_html("<code>{}</code>", plugin["path"]))
-        rows["Hooks"].append(", ".join(plugin["hooks"]) or "(none)")
-
-        # Show config status
-        if plugin.get("config"):
-            config_properties = plugin["config"].get("properties", {})
-            config_count = len(config_properties)
-            rows["Config"].append(f"✅ {config_count} properties" if config_count > 0 else "✅ present")
-        else:
-            rows["Config"].append("❌ none")
-
-    if not plugins:
-        # Show a helpful message when no plugins found
-        rows["Name"].append("(no plugins found)")
-        rows["Source"].append("-")
-        rows["Path"].append(mark_safe("<code>abx_plugins/plugins/</code> or <code>data/custom_plugins/</code>"))
-        rows["Hooks"].append("-")
-        rows["Config"].append("-")
-
-    return TableContext(
-        title="Installed plugins",
-        table=rows,
-    )
-
-
-@render_with_item_view
-def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-    assert is_superuser(request), "Must be a superuser to view configuration settings."
-
-    plugins = get_filesystem_plugins()
-
-    plugin = plugins.get(key)
-    if not plugin:
-        return ItemContext(
-            slug=key,
-            title=f"Plugin not found: {key}",
-            data=[],
-        )
-
-    # Base fields that all plugins have
-    docs_url = get_plugin_docs_url(plugin["name"])
-    machine_admin_url = get_machine_admin_url()
-    fields = {
-        "id": plugin["id"],
-        "name": plugin["name"],
-        "source": plugin["source"],
-    }
-
-    sections: list[SectionData] = [
-        {
-            "name": plugin["name"],
-            "description": format_html(
-                '<code>{}</code><br/><a href="{}" target="_blank" rel="noopener noreferrer">ABX Plugin Docs</a>',
-                plugin["path"],
-                docs_url,
-            ),
-            "fields": fields,
-            "help_texts": {},
-        },
-    ]
-
-    if plugin["hooks"]:
-        sections.append(
-            {
-                "name": "Hooks",
-                "description": mark_safe(render_hook_links_html(plugin["name"], plugin["hooks"], plugin["source"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-    if plugin.get("config"):
-        sections.append(
-            {
-                "name": "Plugin Metadata",
-                "description": mark_safe(render_plugin_metadata_html(plugin["config"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-        sections.append(
-            {
-                "name": "config.json",
-                "description": mark_safe(render_highlighted_json_block(plugin["config"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-        config_properties = plugin["config"].get("properties", {})
-        if config_properties:
-            sections.append(
-                {
-                    "name": "Config Properties",
-                    "description": mark_safe(render_config_properties_html(config_properties, machine_admin_url)),
-                    "fields": {},
-                    "help_texts": {},
-                },
-            )
-
-    return ItemContext(
-        slug=key,
-        title=plugin["name"],
-        data=sections,
-    )
-
-
 @render_with_table_view
 def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."
 
     rows = {
         "Name": [],
+        "Type": [],
         "State": [],
         "PID": [],
         "Started": [],
@@ -657,12 +257,115 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
             continue
         all_config[config_name] = config_data
 
-    # Add top row for supervisord process manager
+    # Collect every PID we plan to show so we can resolve them to Process rows
+    # in a single query. supervisord's per-worker description carries the pid
+    # in the form ``pid 12345, uptime 0:01:23`` (or just the bare ``pid``
+    # placeholder when stopped); we ignore non-numeric values.
+    process_items = supervisor.getAllProcessInfo()
+    if not isinstance(process_items, list):
+        process_items = []
+
+    def _parse_worker_pid_and_uptime(description: str) -> tuple[int | None, str]:
+        body = description.replace("pid ", "", 1)
+        pid_part, _, uptime_part = body.partition(", uptime ")
+        try:
+            return int(pid_part.strip()), uptime_part.strip()
+        except ValueError:
+            return None, ""
+
+    pids: set[int] = set()
+    supervisor_pid = supervisor.getPID()
+    if isinstance(supervisor_pid, int):
+        pids.add(supervisor_pid)
+    for proc_data in process_items:
+        if not isinstance(proc_data, dict):
+            continue
+        pid_int, _ = _parse_worker_pid_and_uptime(str(proc_data.get("description") or ""))
+        if pid_int is not None:
+            pids.add(pid_int)
+
+    pid_to_process_id: dict[int, str] = {}
+    pid_to_process_type: dict[int, str] = {}
+    if pids:
+        try:
+            from archivebox.machine.models import Machine, Process
+
+            for row in (
+                Process.objects.filter(machine=Machine.current(), pid__in=pids)
+                .order_by("pid", "-started_at", "-created_at")
+                .only("id", "pid", "process_type")
+            ):
+                if row.pid in pid_to_process_id:
+                    continue  # keep the most recent row per PID
+                pid_to_process_id[row.pid] = str(row.id)
+                pid_to_process_type[row.pid] = row.process_type
+
+        except Exception:
+            pass
+
+    def _pid_cell(pid_value: int | None, uptime_str: str = ""):
+        if pid_value is None:
+            return ""
+        pid_text = str(pid_value)
+        process_id = pid_to_process_id.get(pid_value)
+        if process_id:
+            link = format_html('<a href="/admin/machine/process/{}/change/">{}</a>', process_id, pid_text)
+        else:
+            link = format_html("{}", pid_text)
+        if uptime_str:
+            return format_html("{}, uptime {}", link, uptime_str)
+        return link
+
+    # Add top row for supervisord process manager. supervisord exposes its
+    # state + pid over XML-RPC but not its own start time / exit status / uptime,
+    # so we read those from the OS process (or fall back to the Process row
+    # recorded in _record_supervisord_process). Exit status stays blank while
+    # it's RUNNING — supervisord wouldn't be answering RPC if it had exited.
     rows["Name"].append(ItemLink("supervisord", key="supervisord"))
+    rows["Type"].append("supervisord")
     supervisor_state = supervisor.getState()
-    rows["State"].append(str(supervisor_state.get("statename") if isinstance(supervisor_state, dict) else ""))
-    rows["PID"].append(str(supervisor.getPID()))
-    rows["Started"].append("-")
+    state_name = str(supervisor_state.get("statename") if isinstance(supervisor_state, dict) else "")
+    rows["State"].append(state_name)
+
+    supervisor_started = ""
+    supervisor_uptime = ""
+    try:
+        import time as _time
+
+        import psutil
+
+        ps_proc = psutil.Process(supervisor_pid)
+        create_time = ps_proc.create_time()
+        supervisor_started = format_parsed_datetime(create_time)
+        seconds = max(int(_time.time() - create_time), 0)
+        hours, remainder = divmod(seconds, 3600)
+        minutes, secs = divmod(remainder, 60)
+        supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
+    except Exception:
+        try:
+            from archivebox.machine.models import Machine, Process
+
+            row = (
+                Process.objects.filter(
+                    machine=Machine.current(),
+                    process_type=Process.TypeChoices.SUPERVISORD,
+                    pid=supervisor_pid,
+                )
+                .order_by("-started_at")
+                .first()
+            )
+            if row and row.started_at:
+                supervisor_started = row.started_at.strftime("%Y-%m-%d %H:%M:%S")
+                seconds = max(int((timezone.now() - row.started_at).total_seconds()), 0)
+                hours, remainder = divmod(seconds, 3600)
+                minutes, secs = divmod(remainder, 60)
+                supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
+        except Exception:
+            pass
+
+    rows["PID"].append(_pid_cell(supervisor_pid if isinstance(supervisor_pid, int) else None, supervisor_uptime))
+    rows["Started"].append(supervisor_started or "-")
+
     rows["Command"].append("supervisord --configuration=tmp/supervisord.conf")
     rows["Logfile"].append(
         format_html(
@@ -671,12 +374,9 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
             "logs/supervisord.log",
         ),
     )
-    rows["Exit Status"].append("0")
+    rows["Exit Status"].append("" if state_name == "RUNNING" else "-")
 
     # Add a row for each worker process managed by supervisord
-    process_items = supervisor.getAllProcessInfo()
-    if not isinstance(process_items, list):
-        process_items = []
     for proc_data in process_items:
         if not isinstance(proc_data, dict):
             continue
@@ -685,10 +385,15 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
         proc_start = proc_data.get("start")
         proc_logfile = str(proc_data.get("stdout_logfile") or "")
         proc_config = all_config.get(proc_name, {})
+        pid_int, uptime_str = _parse_worker_pid_and_uptime(proc_description)
 
         rows["Name"].append(ItemLink(proc_name, key=proc_name))
+        # Prefer the Process row's process_type when we have one (e.g. "worker",
+        # "hook"); otherwise fall back to the generic "worker" label since
+        # everything in this loop is supervisord-managed.
+        rows["Type"].append(pid_to_process_type.get(pid_int, "worker") if pid_int else "worker")
         rows["State"].append(str(proc_data.get("statename") or ""))
-        rows["PID"].append(proc_description.replace("pid ", ""))
+        rows["PID"].append(_pid_cell(pid_int, uptime_str))
         rows["Started"].append(format_parsed_datetime(proc_start))
         rows["Command"].append(str(proc_config.get("command") or ""))
         rows["Logfile"].append(
@@ -811,7 +516,7 @@ def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
                 f.seek(0)
             last_lines = f.read().decode("utf-8", errors="replace").split("\n")
             non_empty_lines = [line for line in last_lines if line.strip()]
-            rows["Most Recent Lines"].append(non_empty_lines[-1])
+            rows["Most Recent Lines"].append(non_empty_lines[-1] if non_empty_lines else "")
 
     return TableContext(
         title="Debug Log files",

archivebox/core/admin_archiveresults.py

@@ -19,15 +19,13 @@ from django.urls import reverse, resolve
 from django.utils import timezone
 from django.utils.text import smart_split
 
-from archivebox.config import DATA_DIR
-from archivebox.config.common import get_config
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.base_models.admin import BaseModelAdmin
-from archivebox.hooks import get_plugin_icon
-from archivebox.core.host_utils import build_snapshot_url
+from archivebox.plugins.discovery import get_plugin_icon
+from archivebox.plugins.views import LIVE_PLUGIN_BASE_URL
+from archivebox.core.routes_util import build_snapshot_url
 from archivebox.core.widgets import InlineTagEditorWidget
-from archivebox.core.views import LIVE_PLUGIN_BASE_URL
-from archivebox.machine.env_utils import env_to_shell_exports
+from archivebox.machine.env_util import env_to_shell_exports
 
 
 from archivebox.core.models import ArchiveResult, Snapshot
@@ -35,7 +33,7 @@ from archivebox.core.models import ArchiveResult, Snapshot
 
 def _get_replay_source_url(result: ArchiveResult) -> str:
     process = result.process_record
-    return str(getattr(process, "url", None) or result.snapshot.url or "")
+    return str((process.url if process else None) or result.snapshot.url or "")
 
 
 def build_abx_dl_display_command(result: ArchiveResult) -> str:
@@ -51,8 +49,8 @@ def build_abx_dl_display_command(result: ArchiveResult) -> str:
 
 def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str:
     display_command = build_abx_dl_display_command(result)
-    process = getattr(result, "process", None)
-    env_items = env_to_shell_exports(getattr(process, "env", None) or {})
+    process = result.process
+    env_items = env_to_shell_exports(process.env if process else {})
     if config is not None:
         result.snapshot._runtime_config = config
     snapshot_dir = shlex.quote(str(result.pwd or result.snapshot_dir))
@@ -62,7 +60,7 @@ def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str:
 
 
 def get_plugin_admin_url(plugin_name: str) -> str:
-    from archivebox.hooks import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, iter_plugin_dirs
+    from archivebox.plugins.discovery import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, iter_plugin_dirs
 
     plugin_dir = next((path.resolve() for path in iter_plugin_dirs() if path.name == plugin_name), None)
     if plugin_dir:
@@ -77,6 +75,12 @@ def get_plugin_admin_url(plugin_name: str) -> str:
     return f"{LIVE_PLUGIN_BASE_URL}builtin.{quote(plugin_name)}/"
 
 
+def get_process_link_label(process) -> str:
+    if process.pid:
+        return str(process.pid)
+    return str(process.id)[-8:]
+
+
 def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
     """Render a nice inline list view of archive results with status, plugin, output, and actions."""
 
@@ -141,7 +145,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
             process_display = f'''
                 <a href="{reverse("admin:machine_process_change", args=[process.id])}"
                    style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
-                   title="View process">{process.pid or "-"}</a>
+                   title="View process">{get_process_link_label(process)}</a>
             '''
 
         machine_display = "-"
@@ -164,8 +168,8 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         cmd_attr = html.escape(replay_cmd, quote=True)
 
         # Build output link - use embed_path() which checks output_files first
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else None
-        snapshot_id = str(getattr(result, "snapshot_id", ""))
+        embed_path = result.embed_path()
+        snapshot_id = str(result.snapshot_id)
         if embed_path and result.status == "succeeded":
             output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
         else:
@@ -354,8 +358,7 @@ class ArchiveResultInline(admin.TabularInline):
     def get_formset(self, request, obj=None, **kwargs):
         formset = super().get_formset(request, obj, **kwargs)
         snapshot = self.get_parent_object_from_request(request)
-        form_class = getattr(formset, "form", None)
-        base_fields = getattr(form_class, "base_fields", {})
+        base_fields = formset.form.base_fields
         snapshot_output_dir = str(snapshot.output_dir) if snapshot else ""
 
         # import ipdb; ipdb.set_trace()
@@ -491,14 +494,12 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     def change_view(self, request, object_id, form_url="", extra_context=None):
         self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         return super().change_view(request, object_id, form_url, extra_context)
 
     def changelist_view(self, request, extra_context=None):
         self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         saved_list_per_page = self.list_per_page
-        self.list_per_page = min(max(5, request.archivebox_config.SNAPSHOTS_PER_PAGE), 5000)
+        self.list_per_page = request.archivebox_config.SNAPSHOTS_PER_PAGE
         try:
             return super().changelist_view(request, extra_context)
         finally:
@@ -522,7 +523,6 @@ class ArchiveResultAdmin(BaseModelAdmin):
             super()
             .get_queryset(request)
             .defer(
-                "config",
                 "notes",
                 "output_json",
             )
@@ -573,20 +573,20 @@ class ArchiveResultAdmin(BaseModelAdmin):
         return queryset.filter(reduce(and_, filters)).distinct(), True
 
     def get_snapshot_view_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        return build_snapshot_url(str(result.snapshot_id), request=request, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return build_snapshot_url(str(result.snapshot_id), request=request, config=request.archivebox_config)
 
     def get_output_view_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
-        output_path = result.embed_path() if hasattr(result, "embed_path") else None
+        request = self.request
+        config = request.archivebox_config
+        output_path = result.embed_path()
         if not output_path:
             output_path = result.plugin or ""
         return build_snapshot_url(str(result.snapshot_id), output_path, request=request, config=config)
 
     def get_output_files_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
+        request = self.request
+        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=request.archivebox_config)}/?files=1"
 
     def get_output_zip_url(self, result: ArchiveResult) -> str:
         return f"{self.get_output_files_url(result)}&download=zip"
@@ -612,10 +612,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
     )
     def snapshot_info(self, result):
         snapshot_id = str(result.snapshot_id)
-        request = getattr(self, "request", None)
+        request = self.request
         return format_html(
             '<a href="{}"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
             snapshot_id[:8],
             result.snapshot.bookmarked_at.strftime("%Y-%m-%d %H:%M"),
             result.snapshot.url[:128],
@@ -665,7 +665,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
         process = result.process_record
         if not process:
             return "-"
-        process_label = process.pid or "-"
+        process_label = get_process_link_label(process)
         return format_html(
             '<a href="{}"><code>{}</code></a>',
             reverse("admin:machine_process_change", args=[process.id]),
@@ -679,17 +679,16 @@ class ArchiveResultAdmin(BaseModelAdmin):
             return "-"
         machine = process.machine
         return format_html(
-            '<a href="{}"><code>{}</code> {}</a>',
+            '<a href="{}">{}</a>',
             reverse("admin:machine_machine_change", args=[machine.id]),
-            str(machine.id)[:8],
             machine.hostname,
         )
 
     @admin.display(description="Command")
     def cmd_str(self, result):
-        request = getattr(self, "request", None)
+        request = self.request
         display_cmd = build_abx_dl_display_command(result)
-        replay_cmd = build_abx_dl_replay_command(result, config=getattr(request, "archivebox_config", None))
+        replay_cmd = build_abx_dl_replay_command(result, config=request.archivebox_config)
         return format_html(
             """
             <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
@@ -710,10 +709,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
         )
 
     def output_display(self, result):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         # Determine output link path - use embed_path() which checks output_files
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else None
+        embed_path = result.embed_path()
         output_path = embed_path if (result.status == "succeeded" and embed_path) else "index.html"
         snapshot_id = str(result.snapshot_id)
         return format_html(
@@ -728,12 +727,12 @@ class ArchiveResultAdmin(BaseModelAdmin):
         if not output_text:
             return "-"
 
-        request = getattr(self, "request", None)
-        live_path = result.embed_path() if hasattr(result, "embed_path") else None
+        request = self.request
+        live_path = result.embed_path()
         if live_path:
             return format_html(
                 '<a href="{}" title="{}"><code>{}</code></a>',
-                build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=getattr(request, "archivebox_config", None)),
+                build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=request.archivebox_config),
                 output_text,
                 output_text,
             )
@@ -785,18 +784,18 @@ class ArchiveResultAdmin(BaseModelAdmin):
         )
 
     def output_summary(self, result):
-        snapshot_dir = Path(DATA_DIR) / str(result.pwd).split("data/", 1)[-1]
+        snapshot_dir = Path(result.snapshot.output_dir)
         output_html = format_html(
             '<pre style="display: inline-block">{}</pre><br/>',
             result.output_str_for_display(),
         )
         snapshot_id = str(result.snapshot_id)
-        request = getattr(self, "request", None)
+        request = self.request
         output_html += format_html(
             '<a href="{}#all">See result files ...</a><br/><pre><code>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
         )
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else ""
+        embed_path = result.embed_path() or ""
         path_from_embed = snapshot_dir / (embed_path or "")
         output_html += format_html(
             '<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>',

archivebox/core/admin_snapshots.py

@@ -1,20 +1,13 @@
 __package__ = "archivebox.core"
 
-import asyncio
 import json
-import threading
-from copy import copy
 from functools import lru_cache
-from queue import Full, Queue
 from types import SimpleNamespace
-from urllib.parse import urlsplit
-from uuid import UUID
 
 from django.contrib import admin, messages
 from django.urls import path, reverse
 from django.shortcuts import get_object_or_404, redirect
-from django.core.cache import cache
-from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed, QueryDict, StreamingHttpResponse
+from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed
 from django.utils import timezone
 from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
@@ -27,35 +20,33 @@ from archivebox.config.common import get_config
 from archivebox.misc.util import htmldecode, urldecode
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.misc.logging_util import printable_filesize
-from archivebox.search.admin import SEARCH_RESULT_CACHE_TTL, SearchResultsAdminMixin, SearchResultsChangeList, get_admin_search_cache_key
-from archivebox.core.host_utils import build_snapshot_url, build_web_url
-from archivebox.core.tag_utils import get_or_create_tag
-from archivebox.hooks import discover_hooks, get_plugin_icon, get_plugin_name, get_plugins
+from archivebox.search.admin import SearchResultsAdminMixin, SearchResultsChangeList
+from archivebox.search.views import admin_snapshot_search_stream_view
+from archivebox.core.routes_util import build_snapshot_url, build_web_url
+from archivebox.core.tag_util import get_or_create_tag
+from archivebox.plugins.hooks import discover_hooks
+from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_plugins
 
 from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 
 from archivebox.core.models import Tag, Snapshot, ArchiveResult
+from archivebox.crawls.models import Crawl
 from archivebox.core.admin_archiveresults import render_archiveresults_list
+from archivebox.core.preview_util import EXTENSION_SCREENSHOT_PLUGIN
+from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
-    PERMISSIONS_PRIVATE,
-    PERMISSIONS_PUBLIC,
-    PERMISSIONS_UNLISTED,
+    PERMISSIONS_META,
     get_snapshot_permissions,
+    normalize_permissions,
 )
 from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget
-from archivebox.crawls.models import Crawl
-from archivebox.personas.models import Persona
 
 
 # GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False}
 GLOBAL_CONTEXT = {}
 
-SNAPSHOT_PERMISSION_META = {
-    PERMISSIONS_PUBLIC: ("👥", "Public", "#047857", "#d1fae5"),
-    PERMISSIONS_UNLISTED: ("🔗", "Unlisted", "#1d4ed8", "#dbeafe"),
-    PERMISSIONS_PRIVATE: ("🔒", "Private", "#991b1b", "#fee2e2"),
-}
+SNAPSHOT_PERMISSION_META = PERMISSIONS_META
 
 
 @lru_cache(maxsize=1)
@@ -118,33 +109,9 @@ class SnapshotPermissionsListFilter(admin.SimpleListFilter):
     def queryset(self, request, queryset):
         value = self.value()
         if value:
-            global_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower()
-            has_overrides = (
-                Snapshot.objects.filter(permissions__gt="").exists()
-                or Crawl.objects.filter(permissions__gt="").exists()
-                or Persona.objects.filter(permissions__gt="").exists()
-            )
-            if not has_overrides:
-                return queryset if value == global_permissions else queryset.none()
-
-            persona_query = Q(crawl__persona_id__in=self.persona_ids_for_value())
-            if global_permissions == value:
-                valid_persona_ids = Persona.objects.values_list("id", flat=True)
-                persona_query |= Q(crawl__persona_id__isnull=True) | ~Q(crawl__persona_id__in=valid_persona_ids)
-            return queryset.filter(
-                Q(permissions=value)
-                | (Q(permissions__isnull=True) & Q(crawl__permissions=value))
-                | (Q(permissions__isnull=True) & Q(crawl__permissions__isnull=True) & persona_query),
-            )
+            return queryset.filter(permissions=value)
         return queryset
 
-    def persona_ids_for_value(self):
-        global_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower()
-        query = Q(permissions=self.value())
-        if global_permissions == self.value():
-            query |= Q(permissions__isnull=True)
-        return Persona.objects.filter(query).values_list("id", flat=True)
-
 
 class SnapshotStatusListFilter(admin.SimpleListFilter):
     title = "snapshot status"
@@ -160,56 +127,6 @@ class SnapshotStatusListFilter(admin.SimpleListFilter):
         return queryset
 
 
-class SnapshotDepthListFilter(admin.SimpleListFilter):
-    title = "depth"
-    parameter_name = "depth_bucket"
-
-    def lookups(self, request, model_admin):
-        return (
-            ("0", "0 root"),
-            ("1", "1"),
-            ("2", "2"),
-            ("3plus", "3+"),
-        )
-
-    def queryset(self, request, queryset):
-        value = self.value()
-        if value == "0":
-            return queryset.filter(depth=0)
-        if value == "1":
-            return queryset.filter(depth=1)
-        if value == "2":
-            return queryset.filter(depth=2)
-        if value == "3plus":
-            return queryset.filter(depth__gte=3)
-        return queryset
-
-
-class SnapshotRelationListFilter(admin.SimpleListFilter):
-    title = "crawl position"
-    parameter_name = "position"
-
-    def lookups(self, request, model_admin):
-        return (
-            ("root", "Root URL"),
-            ("discovered", "Discovered URL"),
-            ("has_children", "Has discovered URLs"),
-            ("no_children", "No discovered URLs"),
-        )
-
-    def queryset(self, request, queryset):
-        value = self.value()
-        if value == "root":
-            return queryset.filter(parent_snapshot__isnull=True)
-        if value == "discovered":
-            return queryset.filter(parent_snapshot__isnull=False)
-        if value in {"has_children", "no_children"}:
-            child_snapshots = Snapshot.objects.filter(parent_snapshot_id=OuterRef("pk"))
-            queryset = queryset.annotate(has_child_snapshots=Exists(child_snapshots))
-            return queryset.filter(has_child_snapshots=value == "has_children")
-        return queryset
-
-
 class SnapshotArchiveStateListFilter(admin.SimpleListFilter):
     title = "archive state"
     parameter_name = "archive_state"
@@ -270,31 +187,10 @@ class SnapshotSizeListFilter(admin.SimpleListFilter):
         return queryset
 
 
-class SnapshotRetryListFilter(admin.SimpleListFilter):
-    title = "retry"
-    parameter_name = "retry"
-
-    def lookups(self, request, model_admin):
-        return (
-            ("due", "Due now"),
-            ("future", "Scheduled later"),
-            ("none", "No retry time"),
-        )
-
-    def queryset(self, request, queryset):
-        value = self.value()
-        if value == "due":
-            return queryset.filter(retry_at__isnull=False, retry_at__lte=timezone.now())
-        if value == "future":
-            return queryset.filter(retry_at__gt=timezone.now())
-        if value == "none":
-            return queryset.filter(retry_at__isnull=True)
-        return queryset
-
-
 class SnapshotResultHealthListFilter(admin.SimpleListFilter):
     title = "ArchiveResult status"
     parameter_name = "archiveresult_status"
+    SNAPSHOT_FIRST_VALUES = {"succeeded"}
 
     def lookups(self, request, model_admin):
         return (
@@ -304,7 +200,6 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
             ("failed", ">50% failed"),
             ("running", ">50% running"),
             ("pending", ">50% queued"),
-            ("backoff", ">50% waiting to retry"),
             ("noresults", ">50% noresults"),
         )
 
@@ -320,60 +215,68 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
                 "succeeded": ArchiveResult.StatusChoices.SUCCEEDED,
                 "failed": ArchiveResult.StatusChoices.FAILED,
                 "running": ArchiveResult.StatusChoices.STARTED,
-                "pending": ArchiveResult.StatusChoices.QUEUED,
+                "pending": (ArchiveResult.StatusChoices.QUEUED, ArchiveResult.StatusChoices.BACKOFF),
                 "backoff": ArchiveResult.StatusChoices.BACKOFF,
                 "noresults": ArchiveResult.StatusChoices.NORESULTS,
             }
             if value in status_by_value:
-                queryset = queryset.annotate(
-                    total_results=Count("archiveresult"),
-                    matching_results=Count(
-                        "archiveresult",
-                        filter=Q(archiveresult__status=status_by_value[value]),
-                    ),
-                )
-                return queryset.filter(matching_results__gt=F("total_results") / 2)
+                status = status_by_value[value]
+                if value in self.SNAPSHOT_FIRST_VALUES:
+                    # "succeeded" is overwhelmingly common in large collections.
+                    # Scan Snapshots in admin order and do indexed per-snapshot
+                    # probes so page 1 can stop after list_per_page matches.
+                    queryset = queryset.alias(
+                        total_results=ArchiveResult.snapshot_count_expr(),
+                        matching_results=ArchiveResult.snapshot_count_expr(status=status),
+                    ).filter(matching_results__gt=F("total_results") / 2)
+                    return queryset
+
+                # Rare statuses are faster status-first: use the
+                # (status, snapshot_id) index to find candidate snapshots.
+                snapshot_ids = ArchiveResult.cached_snapshot_ids_with_majority_status(status)
+                queryset = queryset.filter(pk__in=snapshot_ids)
+                queryset._archivebox_count_hint = len(snapshot_ids)
+                queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
+                return queryset
         return queryset
 
 
 class SnapshotChangeList(SearchResultsChangeList):
     def __init__(self, request, *args, **kwargs):
         super().__init__(request, *args, **kwargs)
-        resolver_name = getattr(getattr(request, "resolver_match", None), "url_name", "")
+        resolver_name = request.resolver_match.url_name
         self.embedded_changelist = request.GET.get("_embedded") == "crawl"
         self.snapshot_is_grid_view = not self.embedded_changelist and (
             resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")
         )
 
+    def _attach_archiveresult_summaries(self):
+        snapshot_ids = [obj.pk for obj in self.result_list]
+        if not snapshot_ids:
+            return
+
+        results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
+        seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids}
+        rows = (
+            ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
+            .order_by("snapshot_id", "plugin")
+            .values_list("snapshot_id", "plugin", "status", "output_size")
+        )
+        for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000):
+            if plugin in seen_plugins[snapshot_id]:
+                continue
+            seen_plugins[snapshot_id].add(plugin)
+            results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size))
+
+        for obj in self.result_list:
+            obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
+
     def get_results(self, request):
         super().get_results(request)
         if request.GET.get("_embedded") == "crawl":
             self.full_result_count = self.result_count
-        else:
-            self.full_result_count = self.model_admin.get_paginator(
-                request,
-                self.model._default_manager.all().order_by(),
-                self.list_per_page,
-            ).count
-        self.show_full_result_count = True
-
-        snapshot_ids = [obj.pk for obj in self.result_list]
-        if snapshot_ids:
-            results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
-            seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids}
-            rows = (
-                ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
-                .order_by("snapshot_id", "plugin")
-                .values_list("snapshot_id", "plugin", "status", "output_size")
-            )
-            for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000):
-                if plugin in seen_plugins[snapshot_id]:
-                    continue
-                seen_plugins[snapshot_id].add(plugin)
-                results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size))
-
-            for obj in self.result_list:
-                obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
+            self.show_full_result_count = True
+        self._attach_archiveresult_summaries()
 
 
 class SnapshotAdminForm(forms.ModelForm):
@@ -408,20 +311,14 @@ class SnapshotAdminForm(forms.ModelForm):
     def save(self, commit=True):
         instance = super().save(commit=False)
         permissions = self.cleaned_data["permissions_config"]
-        inherited_permissions = str(get_config(crawl=instance.crawl, resolve_plugins=False).PERMISSIONS).strip().lower()
         config = dict(instance.config or {})
-        if permissions == inherited_permissions:
-            config.pop("PERMISSIONS", None)
-        else:
-            config["PERMISSIONS"] = permissions
+        config["PERMISSIONS"] = permissions
         instance.config = config
 
         # Handle tags_editor field
         if commit:
             instance.save()
-            save_m2m = getattr(self, "_save_m2m", None)
-            if callable(save_m2m):
-                save_m2m()
+            self._save_m2m()
 
             # Parse and save tags from tags_editor
             tags_str = self.cleaned_data.get("tags_editor", "")
@@ -475,11 +372,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         SnapshotPermissionsListFilter,
         SnapshotStatusListFilter,
         SnapshotResultHealthListFilter,
-        SnapshotDepthListFilter,
-        SnapshotRelationListFilter,
         SnapshotArchiveStateListFilter,
         SnapshotSizeListFilter,
-        SnapshotRetryListFilter,
         "created_at",
         "downloaded_at",
         "crawl__created_by",
@@ -560,8 +454,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         ),
     )
 
-    ordering = ["-timestamp"]
-    actions = ["add_tags", "remove_tags", "resnapshot_snapshot", "update_snapshots", "overwrite_snapshots", "delete_snapshots"]
+    ordering = ["-created_at"]
+    actions = [
+        "add_tags",
+        "remove_tags",
+        "resnapshot_snapshot",
+        "update_snapshots",
+        "overwrite_snapshots",
+        "set_snapshot_permissions",
+        "delete_snapshots",
+    ]
     inlines = []  # Removed TagInline, using TagEditorWidget instead
     list_per_page = 50
 
@@ -569,7 +471,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     paginator = AcceleratedPaginator
 
     save_on_top = True
-    show_full_result_count = False
+    show_full_result_count = True
 
     def get_changelist(self, request, **kwargs):
         return SnapshotChangeList
@@ -580,14 +482,21 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return super().get_ordering(request)
 
     def change_view(self, request, object_id, form_url="", extra_context=None):
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
+        self.request = request
         extra_context = extra_context or {}
         extra_context["CONFIG"] = request.archivebox_config
+        snapshot = self.get_object(request, object_id)
+        if snapshot and snapshot.status in {
+            Snapshot.StatusChoices.QUEUED,
+            Snapshot.StatusChoices.STARTED,
+            Snapshot.StatusChoices.PAUSED,
+        }:
+            extra_context["progress_auto_expand"] = True
+            extra_context["progress_endpoint"] = progress_endpoint("snapshot", snapshot.id)
         return super().change_view(request, object_id, form_url, extra_context | GLOBAL_CONTEXT)
 
     def changelist_view(self, request, extra_context=None):
         self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         saved_list_per_page = self.list_per_page
         embedded_changelist = request.GET.get("_embedded") == "crawl"
         if embedded_changelist:
@@ -597,7 +506,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                 requested_per_page = 200
             self.list_per_page = min(max(200, requested_per_page), 500)
         else:
-            self.list_per_page = min(max(50, request.archivebox_config.SNAPSHOTS_PER_PAGE), 500)
+            self.list_per_page = request.archivebox_config.SNAPSHOTS_PER_PAGE
         extra_context = extra_context or {}
         extra_context["embedded_changelist"] = embedded_changelist
         extra_context["CONFIG"] = request.archivebox_config
@@ -623,12 +532,12 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return super().lookup_allowed(lookup, value, request=request)
 
     def get_snapshot_view_url(self, obj: Snapshot) -> str:
-        request = getattr(self, "request", None)
-        return build_snapshot_url(str(obj.id), request=request, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)
 
     def get_snapshot_files_url(self, obj: Snapshot) -> str:
-        request = getattr(self, "request", None)
-        return f"{build_snapshot_url(str(obj.id), request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
+        request = self.request
+        return f"{build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)}/?files=1"
 
     def get_snapshot_zip_url(self, obj: Snapshot) -> str:
         return f"{self.get_snapshot_files_url(obj)}&download=zip"
@@ -648,155 +557,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return custom_urls + urls
 
     def search_stream_view(self, request):
-        from archivebox.search import iter_query_search_ids
-
-        query = (request.GET.get("q") or "").strip()
-        from archivebox.search import get_search_mode, get_search_mode_base
-
-        search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
-        if not query:
-            return StreamingHttpResponse((), content_type="text/plain")
-
-        search_url = request.GET.get("search_url") or request.get_full_path()
-        target_url = urlsplit(search_url)
-        target_get = QueryDict(target_url.query, mutable=True)
-        for key in ("q", "search_mode", "p", "search_url"):
-            target_get.pop(key, None)
-
-        filter_request = copy(request)
-        filter_request.path = target_url.path or request.path
-        filter_request.path_info = target_url.path or request.path_info
-        filter_request.GET = target_get
-        filter_request.archivebox_config = getattr(request, "archivebox_config", None)
-
-        # Build the same filtered base queryset the changelist uses, but with
-        # the search params stripped. The stream then intersects each wave with
-        # this queryset before writing IDs into the short-lived cache.
-        current_request = getattr(self, "request", None)
-        try:
-            base_queryset = self.get_changelist_instance(filter_request).queryset
-        finally:
-            self.request = current_request
-
-        async def snapshot_ids():
-            seen = set()
-            ids = []
-            last_sent = 0
-            stream_batch_size = 100
-            stream_padding = " " * 4096
-            cache_key = get_admin_search_cache_key(request, search_url)
-            cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-            yield f"0{stream_padding}\n"
-            queue = Queue(maxsize=8)
-            stop_event = threading.Event()
-
-            def emit(item):
-                while not stop_event.is_set():
-                    try:
-                        queue.put(item, timeout=0.1)
-                        return
-                    except Full:
-                        continue
-
-            def run_search():
-                nonlocal last_sent
-                iterator = None
-                try:
-                    search_mode_base = get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None))
-                    iterator = (
-                        self.iter_meta_search_ids(query, base_queryset)
-                        if search_mode_base == "meta"
-                        else self.iter_backend_search_ids(
-                            iter_query_search_ids(query, search_mode=search_mode, config=getattr(request, "archivebox_config", None)),
-                            base_queryset,
-                        )
-                    )
-                    for snapshot_id in iterator:
-                        if stop_event.is_set():
-                            break
-                        snapshot_id = str(snapshot_id).strip().lower()
-                        if len(snapshot_id.replace("-", "")) != 32 or snapshot_id in seen:
-                            continue
-                        seen.add(snapshot_id)
-                        ids.append(snapshot_id)
-                        if len(ids) - last_sent >= stream_batch_size:
-                            cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-                            last_sent = len(ids)
-                            emit(f"{last_sent}{stream_padding}\n")
-                    if not stop_event.is_set() and len(ids) != last_sent:
-                        cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-                        emit(f"{len(ids)}{stream_padding}\n")
-                except BaseException as err:
-                    emit(err)
-                finally:
-                    if iterator is not None:
-                        try:
-                            iterator.close()
-                        except AttributeError:
-                            pass
-                    cache.set(cache_key, {"ids": ids, "done": True}, SEARCH_RESULT_CACHE_TTL)
-                    emit(None)
-
-            threading.Thread(target=run_search, name="admin-snapshot-search-stream", daemon=True).start()
-            try:
-                while True:
-                    item = await asyncio.to_thread(queue.get)
-                    if item is None:
-                        break
-                    if isinstance(item, BaseException):
-                        raise item
-                    yield item
-            finally:
-                stop_event.set()
-
-        response = StreamingHttpResponse(snapshot_ids(), content_type="text/plain")
-        response["X-Accel-Buffering"] = "no"
-        return response
-
-    def iter_meta_search_ids(self, query, queryset):
-        seen = set()
-        try:
-            snapshot_id = UUID(query)
-        except ValueError:
-            snapshot_id = None
-        if snapshot_id:
-            for pk in queryset.filter(pk=snapshot_id).values_list("pk", flat=True):
-                seen.add(pk)
-                yield pk
-
-        for wave in (
-            Q(timestamp__startswith=query) | Q(url__istartswith=query) | Q(title__istartswith=query),
-            Q(url__icontains=query),
-            Q(title__icontains=query),
-            Q(tags__name__icontains=query),
-        ):
-            for pk in queryset.filter(wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500):
-                if pk in seen:
-                    continue
-                seen.add(pk)
-                yield pk
-
-    def iter_backend_search_ids(self, iterator, queryset):
-        batch = []
-        seen = set()
-
-        def flush_batch():
-            valid = {str(pk) for pk in queryset.filter(pk__in=batch).values_list("pk", flat=True)}
-            for snapshot_id in batch:
-                if snapshot_id in valid and snapshot_id not in seen:
-                    seen.add(snapshot_id)
-                    yield snapshot_id
-
-        for snapshot_id in iterator:
-            snapshot_id = str(snapshot_id).strip().lower()
-            if len(snapshot_id.replace("-", "")) != 32:
-                continue
-            batch.append(snapshot_id)
-            if len(batch) >= 200:
-                yield from flush_batch()
-                batch = []
-        if batch:
-            yield from flush_batch()
+        return admin_snapshot_search_stream_view(self, request)
 
     def set_permissions_view(self, request, object_id):
         if request.method != "POST":
@@ -806,13 +567,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         if permissions not in dict(PERMISSIONS_CHOICES):
             return HttpResponseBadRequest("Invalid permissions value")
 
-        snapshot = get_object_or_404(Snapshot.objects.select_related("crawl"), pk=object_id)
+        snapshot = get_object_or_404(Snapshot, pk=object_id)
         config = dict(snapshot.config or {})
-        inherited_permissions = str(get_config(crawl=snapshot.crawl, resolve_plugins=False).PERMISSIONS).strip().lower()
-        if permissions == inherited_permissions:
-            config.pop("PERMISSIONS", None)
-        else:
-            config["PERMISSIONS"] = permissions
+        config["PERMISSIONS"] = permissions
 
         # Keep the quick-edit write to one targeted UPDATE so SQLite only holds
         # the write lock for the permission/config change itself. safe_update()
@@ -822,6 +579,35 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
         return JsonResponse({"permissions": permissions, "icon": icon, "label": label, "fg": fg, "bg": bg})
 
+    @admin.action(description="Permissions ▾")
+    def set_snapshot_permissions(self, request, queryset):
+        permissions = (request.POST.get("permissions") or "").strip().lower()
+        if permissions not in dict(PERMISSIONS_CHOICES):
+            messages.error(request, "Choose a valid permissions value.")
+            return
+        updated = self.update_snapshot_permissions(queryset, permissions)
+        messages.success(request, f"Set permissions to {permissions} on {updated} snapshot(s).")
+
+    def update_snapshot_permissions(self, queryset, permissions):
+        now = timezone.now()
+        updated = 0
+        batch = []
+        snapshots = queryset.select_related(None).only("id", "config").prefetch_related(None)
+        for snapshot in snapshots.iterator(chunk_size=500):
+            config = dict(snapshot.config or {})
+            config["PERMISSIONS"] = permissions
+            snapshot.config = config
+            snapshot.modified_at = now
+            batch.append(snapshot)
+            if len(batch) >= 500:
+                Snapshot.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
+                updated += len(batch)
+                batch.clear()
+        if batch:
+            Snapshot.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
+            updated += len(batch)
+        return updated
+
     def redo_failed_view(self, request, object_id):
         snapshot = get_object_or_404(Snapshot, pk=object_id)
 
@@ -840,33 +626,13 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
         return redirect(snapshot.admin_change_url)
 
-    # def get_queryset(self, request):
-    #     # tags_qs = SnapshotTag.objects.all().select_related('tag')
-    #     # prefetch = Prefetch('snapshottag_set', queryset=tags_qs)
-
-    #     self.request = request
-    #     return super().get_queryset(request).prefetch_related('archiveresult_set').distinct()  # .annotate(archiveresult_count=Count('archiveresult'))
     def get_queryset(self, request):
         self.request = request
         ordering_fields = self._get_ordering_fields(request)
         needs_files_sort = "files" in ordering_fields
         needs_tags_sort = "tags_inline" in ordering_fields
-        is_change_view = getattr(getattr(request, "resolver_match", None), "url_name", "") == "core_snapshot_change"
-        request.archivebox_default_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower()
-
-        prefetches = [
-            Prefetch(
-                "crawl",
-                queryset=Crawl.objects.only(
-                    "id",
-                    "permissions",
-                    "persona_id",
-                    "status",
-                    "created_by_id",
-                ).prefetch_related("created_by"),
-            ),
-            "tags",
-        ]
+        is_change_view = request.resolver_match.url_name == "core_snapshot_change"
+        prefetches = ["tags"]
         if is_change_view:
             prefetches.append(
                 Prefetch(
@@ -877,13 +643,26 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                         "plugin",
                         "status",
                         "output_size",
+                        "output_files",
+                    ),
+                ),
+            )
+        else:
+            prefetches.append(
+                Prefetch(
+                    "crawl",
+                    queryset=Crawl.objects.select_related("created_by").only(
+                        "id",
+                        "created_by_id",
+                        "created_by__id",
+                        "created_by__username",
                     ),
                 ),
             )
 
         qs = super().get_queryset(request)
         if is_change_view:
-            qs = qs.defer("notes")
+            qs = qs.select_related("crawl__created_by").defer("notes")
         else:
             qs = qs.only(
                 "id",
@@ -901,10 +680,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         qs = qs.prefetch_related(*prefetches)
         if needs_files_sort:
             qs = qs.annotate(
-                ar_succeeded_count=Count(
-                    "archiveresult",
-                    filter=Q(archiveresult__status="succeeded"),
-                ),
+                ar_succeeded_count=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
             )
         if needs_tags_sort:
             qs = qs.annotate(tag_count=Count("tags", distinct=True))
@@ -913,24 +689,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="👁", ordering="permissions")
     def permissions_badge(self, obj):
-        request = self.request
-        permissions = getattr(obj, "snapshot_permissions", None)
+        permissions = obj.__dict__.get("snapshot_permissions")
         if permissions is None:
-            if obj.permissions:
-                permissions = obj.permissions
-            elif obj.crawl.permissions:
-                permissions = obj.crawl.permissions
-            elif obj.crawl.persona_id:
-                persona_permissions = getattr(request, "archivebox_persona_permissions", None)
-                if persona_permissions is None:
-                    persona_permissions = {
-                        str(persona.id): persona.permissions or request.archivebox_default_permissions
-                        for persona in Persona.objects.only("id", "permissions")
-                    }
-                    request.archivebox_persona_permissions = persona_permissions
-                permissions = persona_permissions.get(str(obj.crawl.persona_id), request.archivebox_default_permissions)
-            else:
-                permissions = request.archivebox_default_permissions
+            permissions = obj.permissions
+        permissions = normalize_permissions(permissions)
         icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
         menu_items = format_html_join(
             "",
@@ -1094,8 +856,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
 
     def status_info(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         favicon_url = build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)
         return format_html(
             """
@@ -1112,16 +874,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Archive Results")
     def archiveresults_list(self, obj):
-        request = getattr(self, "request", None)
-        return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=request.archivebox_config)
 
     @admin.display(
         description="Title",
         ordering="title",
     )
     def title_str(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         title_raw = (obj.title or "").strip()
         url_raw = (obj.url or "").strip()
         title_normalized = title_raw.lower()
@@ -1174,27 +936,35 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return mark_safe(f'<span class="tags-inline-editor">{tags_html}</span>')
 
     def _get_preview_data(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         results = self._get_prefetched_results(obj)
         if results is not None:
             has_screenshot = any(r.plugin == "screenshot" for r in results)
             has_favicon = any(r.plugin == "favicon" for r in results)
+            has_extension_screenshot = any(r.plugin == EXTENSION_SCREENSHOT_PLUGIN for r in results)
         else:
-            available_plugins = set(obj.archiveresult_set.filter(plugin__in=("screenshot", "favicon")).values_list("plugin", flat=True))
+            available_plugins = set(
+                obj.archiveresult_set.filter(plugin__in=("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")).values_list(
+                    "plugin",
+                    flat=True,
+                ),
+            )
             has_screenshot = "screenshot" in available_plugins
             has_favicon = "favicon" in available_plugins
+            has_extension_screenshot = EXTENSION_SCREENSHOT_PLUGIN in available_plugins
 
-        if not has_screenshot and not has_favicon:
+        extension_screenshot_urls = [
+            build_snapshot_url(str(obj.id), f"{EXTENSION_SCREENSHOT_PLUGIN}/screenshot-1.png", request=request, config=config),
+            build_snapshot_url(str(obj.id), f"{EXTENSION_SCREENSHOT_PLUGIN}/screenshot.png", request=request, config=config),
+        ]
+
+        if not has_screenshot and not has_extension_screenshot and not has_favicon:
             return None
 
-        if has_screenshot:
+        if has_screenshot or has_extension_screenshot:
             img_url = build_snapshot_url(str(obj.id), "screenshot/screenshot.png", request=request, config=config)
-            fallbacks = [
-                build_snapshot_url(str(obj.id), "screenshot.png", request=request, config=config),
-                build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config),
-                build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config),
-            ]
+            fallbacks = extension_screenshot_urls
             img_alt = "Screenshot"
             preview_class = "screenshot"
         else:
@@ -1227,8 +997,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         if not preview:
             return ""
 
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         favicon_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
         fallback_list = ",".join([build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)])
         onerror_js = (
@@ -1266,8 +1036,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description=" ", empty_value="")
     def snapshot_summary(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         preview = self._get_preview_data(obj)
         stats = self._get_progress_stats(obj)
         archive_size = stats["output_size"] or 0
@@ -1331,8 +1101,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
         visible_results = sorted_results[:14]
         output = []
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         for result in visible_results:
             icon = mark_safe(get_plugin_icon(result.plugin))
             if not icon.strip():
@@ -1363,8 +1133,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         # ordering='archiveresult_count'
     )
     def size(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         archive_size = self._get_progress_stats(obj)["output_size"] or 0
         if archive_size:
             size_txt = printable_filesize(archive_size)
@@ -1478,7 +1248,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
 
     def _get_progress_stats(self, obj):
-        cached_stats = getattr(obj, "_admin_progress_stats", None)
+        cached_stats = obj.__dict__.get("_admin_progress_stats")
         if cached_stats is not None:
             return cached_stats
 
@@ -1524,39 +1294,38 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def _get_prefetched_results(self, obj):
         if "_admin_archiveresults" in obj.__dict__:
             return obj.__dict__["_admin_archiveresults"]
-        if hasattr(obj, "_prefetched_objects_cache") and "archiveresult_set" in obj._prefetched_objects_cache:
+        if "archiveresult_set" in obj.__dict__.get("_prefetched_objects_cache", {}):
             return obj.archiveresult_set.all()
         return None
 
     def _get_expected_hook_total(self, obj) -> int:
         try:
-            request = getattr(self, "request", None)
-            if getattr(getattr(request, "resolver_match", None), "url_name", "") in {"core_snapshot_changelist", "core_snapshot_change"}:
+            request = self.request
+            if request.resolver_match.url_name in {"core_snapshot_changelist", "core_snapshot_change"}:
                 return 0
 
-            crawl = getattr(obj, "crawl", None)
-            snapshot_config = getattr(obj, "config", None) or {}
-            crawl_config = getattr(crawl, "config", None) or {}
-            crawl_persona_id = getattr(crawl, "persona_id", None)
-            has_scoped_config = bool(snapshot_config or crawl_config or crawl_persona_id)
+            crawl = obj.crawl
+            snapshot_config = obj.config or {}
+            crawl_config = crawl.config or {}
+            has_scoped_config = bool(snapshot_config or crawl_config)
 
             if request is not None and not has_scoped_config:
-                cached_total = getattr(request, "archivebox_expected_snapshot_hook_total", None)
+                cached_total = request.__dict__.get("archivebox_expected_snapshot_hook_total")
                 if cached_total is None:
-                    config = getattr(request, "archivebox_config", None) or get_config()
+                    config = request.archivebox_config
                     cached_total = len(discover_hooks("Snapshot", config=config))
                     request.archivebox_expected_snapshot_hook_total = cached_total
                 return cached_total
 
             if request is not None:
-                scoped_cache = getattr(request, "archivebox_expected_snapshot_hook_totals_by_scope", None)
+                scoped_cache = request.__dict__.get("archivebox_expected_snapshot_hook_totals_by_scope")
                 if scoped_cache is None:
                     scoped_cache = {}
                     request.archivebox_expected_snapshot_hook_totals_by_scope = scoped_cache
                 if snapshot_config:
                     cache_key = ("snapshot", json.dumps(snapshot_config, sort_keys=True, default=str))
                 else:
-                    cache_key = ("crawl", json.dumps(crawl_config, sort_keys=True, default=str), crawl_persona_id)
+                    cache_key = ("crawl", json.dumps(crawl_config, sort_keys=True, default=str))
                 cached_total = scoped_cache.get(cache_key)
                 if cached_total is None:
                     config = get_config(crawl=crawl, snapshot=obj if snapshot_config else None)
@@ -1569,8 +1338,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             return 0
 
     def _get_prefetched_tags(self, obj):
-        if hasattr(obj, "_prefetched_objects_cache") and "tags" in obj._prefetched_objects_cache:
-            return list(obj._prefetched_objects_cache["tags"])
+        prefetched_cache = obj.__dict__.get("_prefetched_objects_cache", {})
+        if "tags" in prefetched_cache:
+            return list(prefetched_cache["tags"])
         return None
 
     def _get_ordering_fields(self, request):
@@ -1648,13 +1418,23 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
         from archivebox.cli.archivebox_add import add
 
-        add(urls=urls, bg=True)
+        # "Archive Now" is an explicit user re-archive — force ONLY_NEW=False
+        # on the resulting crawl so existing snapshots don't cause the crawl to
+        # seal immediately with zero new snapshots (the default ONLY_NEW=True
+        # would skip any URLs that have ever been archived before).
+        crawl, _ = add(urls=urls, bg=True, config={"ONLY_NEW": False})
 
         messages.success(
             request,
             f"Created 1 queued crawl with {len(snapshots)} URL(s). The background runner will create snapshots and process them.",
         )
 
+        # Redirect to the new crawl's admin page so the user lands on the
+        # work-in-progress crawl, not the old snapshot they re-archived from.
+        # A snapshot-view redirect would race the runner — the new snapshot
+        # may sit queued for a while before the runner creates the DB row.
+        return redirect(f"/admin/crawls/crawl/{crawl.id}/change/#snapshots")
+
     @admin.action(
         description="🔄 Redo",
     )

archivebox/core/admin_tags.py

@@ -13,7 +13,7 @@ from django.utils.safestring import mark_safe
 
 from archivebox.base_models.admin import BaseModelAdmin
 from archivebox.core.models import SnapshotTag, Tag
-from archivebox.core.tag_utils import (
+from archivebox.core.tag_util import (
     TAG_HAS_SNAPSHOTS_CHOICES,
     TAG_SORT_CHOICES,
     build_tag_cards,
@@ -24,7 +24,7 @@ from archivebox.core.tag_utils import (
     normalize_has_snapshots_filter,
     normalize_tag_sort,
 )
-from archivebox.core.host_utils import build_snapshot_url
+from archivebox.core.routes_util import build_snapshot_url
 
 
 class TagInline(admin.TabularInline):
@@ -117,7 +117,7 @@ class TagAdmin(BaseModelAdmin):
         sort = normalize_tag_sort((request.GET.get("sort") or "created_desc").strip())
         created_by = normalize_created_by_filter((request.GET.get("created_by") or "").strip())
         year = normalize_created_year_filter((request.GET.get("year") or "").strip())
-        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "yes").strip())
+        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "all").strip())
         context = {
             **self.admin_site.each_context(request),
             **(extra_context or {}),
@@ -231,7 +231,7 @@ class TagAdmin(BaseModelAdmin):
 
     @admin.display(description="Snapshots", ordering="num_snapshots")
     def num_snapshots(self, tag: Tag):
-        count = getattr(tag, "num_snapshots", None)
+        count = tag.__dict__.get("num_snapshots")
         if count is None:
             count = tag.snapshot_set.count()
         return format_html(

archivebox/core/admin_users.py

@@ -47,7 +47,7 @@ class CustomUserAdmin(UserAdmin):
 
     def snapshot_count_badge(self, obj):
         snapshots_url = f"/admin/core/snapshot/?created_by__id__exact={obj.pk}"
-        snapshot_count = getattr(obj, "snapshot_count", 0)
+        snapshot_count = obj.__dict__.get("snapshot_count", 0)
         snapshot_label = "snapshot" if snapshot_count == 1 else "snapshots"
         return format_html(
             (

archivebox/core/apps.py

@@ -41,5 +41,5 @@ class CoreConfig(AppConfig):
                 worker_type="worker_runserver",
                 pwd=str(CONSTANTS.DATA_DIR),
                 url=os.environ.get("ARCHIVEBOX_RUNSERVER_BIND_URL") or "",
-                timeout=0,
+                timeout=CONSTANTS.MAX_HOOK_RUNTIME_SECONDS,
             )

archivebox/core/asgi.py

@@ -12,6 +12,65 @@ from django.core.asgi import get_asgi_application
 
 setup_django(in_memory_db=False, check_db=True)
 
+
+def _patch_thread_sensitive_context_shutdown() -> None:
+    """Stop ``ThreadSensitiveContext.__aexit__`` from blocking the daphne loop.
+
+    Django 6.0's ASGIHandler wraps every request in ``async with
+    ThreadSensitiveContext():`` (django/core/handlers/asgi.py:169). On exit
+    asgiref calls ``executor.shutdown()`` with the default ``wait=True``
+    (asgiref/sync.py:148), which is a *synchronous* ``Thread.join()`` inside
+    an async function — so it blocks the daphne event loop until the
+    executor's worker thread exits.
+
+    That's normally fine because the request handler has already awaited
+    every ``sync_to_async`` it submitted, so the worker is idle and dies as
+    soon as the shutdown sentinel reaches it. The blocking turns into a
+    problem when a client disconnects mid-request:
+
+    * ``SyncToAsync.__call__`` shields the executor work with
+      ``await asyncio.shield(exec_coro)`` (asgiref/sync.py:506) so that the
+      sync DB call doesn't get torn down halfway through.
+    * On cancellation it calls ``exec_coro.cancel()`` (line 522) which only
+      flips the asyncio ``Future`` to cancelled — the underlying thread
+      keeps running the SQL query.
+    * Control unwinds to ``__aexit__`` while the orphaned thread is still
+      mid-query. ``shutdown(wait=True)`` then blocks the event loop until
+      that orphan finishes.
+
+    Under heavy SQLite contention (the load-test scenario that surfaced
+    this on cabbage) those orphan threads can take 30 seconds each waiting
+    for write locks, and the daphne loop is single-threaded — so every
+    such orphan stalls every other in-flight request, healthchecks time
+    out, and the container goes ``unhealthy``.
+
+    Switching to ``shutdown(wait=False)`` queues the sentinel and returns
+    immediately; the worker thread still exits cleanly once its current
+    task finishes, and asgiref's ``WeakKeyDictionary`` releases the
+    executor as soon as the request's context is GC'd. No per-request
+    teardown guarantee is lost — there was no caller relying on it.
+    """
+    from asgiref import sync as _asgiref_sync
+
+    original_aexit = _asgiref_sync.ThreadSensitiveContext.__aexit__
+
+    async def __aexit__(self, exc, value, tb):  # type: ignore[no-redef]
+        if not self.token:
+            return
+        executor = _asgiref_sync.SyncToAsync.context_to_thread_executor.pop(self, None)
+        if executor is not None:
+            executor.shutdown(wait=False)
+        _asgiref_sync.SyncToAsync.thread_sensitive_context.reset(self.token)
+
+    # Idempotent: only patch once even if asgi.py is reloaded.
+    if getattr(original_aexit, "_archivebox_patched", False):
+        return
+    __aexit__._archivebox_patched = True  # type: ignore[attr-defined]
+    _asgiref_sync.ThreadSensitiveContext.__aexit__ = __aexit__
+
+
+_patch_thread_sensitive_context_shutdown()
+
 # Standard Django ASGI application (no websockets/channels needed)
 application = get_asgi_application()
 

archivebox/core/context_processors.py

@@ -0,0 +1,9 @@
+from archivebox.config import VERSION
+from archivebox.config.version import get_COMMIT_HASH
+
+
+def archivebox_globals(request):
+    return {
+        "VERSION": VERSION,
+        "STATIC_CACHE_KEY": (get_COMMIT_HASH() or VERSION or "dev").strip(),
+    }

archivebox/core/forms.py

@@ -1,23 +1,24 @@
 __package__ = "archivebox.core"
 
-import json
 import re
-from collections.abc import Iterable, Mapping
 from decimal import Decimal, InvalidOperation, ROUND_CEILING
-from pathlib import Path
-from typing import Any
 
 from django import forms
-from django.utils.html import format_html
 
 from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
 from taggit.utils import edit_string_for_tags, parse_tags
 from archivebox.base_models.admin import KeyValueWidget
-from archivebox.crawls.schedule_utils import validate_schedule
+from archivebox.crawls.schedule_util import validate_schedule
 from archivebox.config.common import get_config, parse_delete_after
 from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions, is_admin_user
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
-from archivebox.hooks import get_plugins, discover_plugin_configs, get_plugin_icon
+from archivebox.plugins.discovery import get_plugins
+from archivebox.plugins.forms import (
+    PLUGIN_GROUP_DEFINITIONS,
+    TIMEOUT_INPUT_PATTERN,
+    PluginConfigFormMixin,
+    get_choice_field,
+)
 from archivebox.personas.models import Persona
 
 DEPTH_CHOICES = (
@@ -28,484 +29,10 @@ DEPTH_CHOICES = (
     ("4", "depth = 4 (+ URLs four hops away)"),
 )
 
-PLUGIN_CONFIG_FIELD_PREFIX = "plugin_config__"
-PLUGIN_GROUP_DEFINITIONS = (
-    (
-        "main_plugins",
-        "Main",
-        "",
-        "",
-        "",
-        (
-            "dom",
-            "screenshot",
-            "pdf",
-            "singlefile",
-            "wget",
-            "archivedotorg",
-            "chrome_mhtml",
-        ),
-    ),
-    (
-        "page_setup_plugins",
-        "Page Setup",
-        "",
-        "",
-        "",
-        (
-            "chrome",
-            "infiniscroll",
-            "modalcloser",
-            "ublock",
-            "istilldontcareaboutcookies",
-            "twocaptcha",
-            "claudechrome",
-        ),
-    ),
-    (
-        "media_plugins",
-        "Media",
-        "",
-        "",
-        "",
-        (
-            "staticfile",
-            "responses",
-            "ytdlp",
-            "gallerydl",
-            "git",
-        ),
-    ),
-    (
-        "text_plugins",
-        "Text",
-        "",
-        "",
-        "",
-        (
-            "readability",
-            "htmltotext",
-            "defuddle",
-            "forumdl",
-            "mercury",
-            "trafilatura",
-            "liteparse",
-            "opendataloader",
-            "papersdl",
-        ),
-    ),
-    (
-        "metadata_plugins",
-        "Metadata",
-        "",
-        "",
-        "",
-        (
-            "title",
-            "favicon",
-            "headers",
-            "redirects",
-            "accessibility",
-            "consolelog",
-            "sslcerts",
-            "dns",
-            "seo",
-            "hashes",
-        ),
-    ),
-    (
-        "postprocessing_plugins",
-        "Postprocessing",
-        "",
-        "",
-        "",
-        (
-            "parse_dom_outlinks",
-            "parse_html_urls",
-            "parse_jsonl_urls",
-            "parse_netscape_urls",
-            "parse_rss_urls",
-            "parse_txt_urls",
-            "claudecode",
-            "claudecodecleanup",
-            "claudecodeextract",
-        ),
-    ),
-)
-HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
-    "apt",
-    "base",
-    "bash",
-    "brew",
-    "cargo",
-    "chromewebstore",
-    "env",
-    "media",
-    "npm",
-    "pip",
-    "puppeteer",
-    "search_backend_ripgrep",
-    "search_backend_sonic",
-    "search_backend_sqlite",
-    "ssl",
-}
-TIMEOUT_INPUT_PATTERN = r"(0|[1-9][0-9]*|[0-9]+(?:\.[0-9]+)?\s*(?:s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours))"
-
-
-def get_plugin_choices():
-    """Get available extractor plugins from discovered hooks."""
-    return [(name, name) for name in get_plugins()]
-
-
-def get_plugin_choice_label(plugin_name: str, plugin_configs: dict[str, dict]) -> str:
-    schema = plugin_configs.get(plugin_name, {})
-    description = str(schema.get("description") or "").strip()
-    if not description:
-        return plugin_name
-    icon_html = get_plugin_icon(plugin_name)
-
-    return format_html(
-        '<span class="plugin-choice-icon">{}</span><span class="plugin-choice-name">{}</span><a class="plugin-choice-description" href="https://archivebox.github.io/abx-plugins/#{}" target="_blank" rel="noopener noreferrer">{}</a>',
-        icon_html,
-        plugin_name,
-        plugin_name,
-        description,
-    )
-
-
-def get_choice_field(form: forms.Form, name: str) -> forms.ChoiceField:
-    field = form.fields[name]
-    if not isinstance(field, forms.ChoiceField):
-        raise TypeError(f"{name} must be a ChoiceField")
-    return field
-
-
-def _plugin_config_input_name(plugin_name: str, config_key: str) -> str:
-    return f"{PLUGIN_CONFIG_FIELD_PREFIX}{plugin_name}__{config_key}"
-
-
-def _schema_types(schema: Mapping[str, Any]) -> list[str]:
-    raw_type = schema.get("type") or "string"
-    if isinstance(raw_type, list):
-        return [str(item) for item in raw_type]
-    return [str(raw_type)]
-
-
-def _jsonish(value: Any) -> str:
-    if isinstance(value, str):
-        return value
-    return json.dumps(value, sort_keys=True, default=str)
-
-
-def _same_config_value(left: Any, right: Any) -> bool:
-    return json.dumps(left, sort_keys=True, default=str) == json.dumps(right, sort_keys=True, default=str)
-
-
-def _coerce_plugin_config_value(raw_value: Any, schema: Mapping[str, Any]) -> Any:
-    schema_types = _schema_types(schema)
-
-    if "boolean" in schema_types:
-        if isinstance(raw_value, bool):
-            return raw_value
-        value = str(raw_value).strip().lower()
-        if value in {"true", "1", "yes", "on"}:
-            return True
-        if value in {"false", "0", "no", "off", ""}:
-            return False
-        raise forms.ValidationError("Must be true or false.")
-
-    if "integer" in schema_types:
-        value = int(str(raw_value).strip())
-        minimum = schema.get("minimum")
-        maximum = schema.get("maximum")
-        if minimum is not None and value < int(minimum):
-            raise forms.ValidationError(f"Must be at least {minimum}.")
-        if maximum is not None and value > int(maximum):
-            raise forms.ValidationError(f"Must be at most {maximum}.")
-        return value
-
-    if "number" in schema_types:
-        value = float(str(raw_value).strip())
-        minimum = schema.get("minimum")
-        maximum = schema.get("maximum")
-        if minimum is not None and value < float(minimum):
-            raise forms.ValidationError(f"Must be at least {minimum}.")
-        if maximum is not None and value > float(maximum):
-            raise forms.ValidationError(f"Must be at most {maximum}.")
-        return value
-
-    if "array" in schema_types:
-        if isinstance(raw_value, list):
-            return raw_value
-        value = str(raw_value).strip()
-        if not value:
-            return []
-        if value.startswith("["):
-            parsed = json.loads(value)
-            if not isinstance(parsed, list):
-                raise forms.ValidationError("Must be a JSON array.")
-            return parsed
-        return [item.strip() for item in value.replace(",", "\n").splitlines() if item.strip()]
-
-    if "object" in schema_types:
-        value = str(raw_value).strip()
-        if not value:
-            return {}
-        parsed = json.loads(value)
-        if not isinstance(parsed, dict):
-            raise forms.ValidationError("Must be a JSON object.")
-        return parsed
-
-    value = str(raw_value)
-    enum = schema.get("enum")
-    if isinstance(enum, list) and enum and value not in {str(item) for item in enum}:
-        raise forms.ValidationError(f"Must be one of: {', '.join(str(item) for item in enum)}.")
-    return value
-
-
-class PluginConfigFormMixin:
-    plugin_groups: list[dict[str, Any]]
-
-    def build_plugin_groups(self, runtime_config: Mapping[str, Any] | None = None) -> None:
-        all_plugins = get_plugins()
-        plugin_configs = discover_plugin_configs()
-        runtime_config = runtime_config or get_config()
-        self.plugin_config_binary_urls = get_plugin_config_binary_urls(runtime_config)
-        grouped_plugins = set().union(*(group[-1] for group in PLUGIN_GROUP_DEFINITIONS))
-        other_plugins = tuple(sorted(set(all_plugins) - grouped_plugins - HIDDEN_PLUGIN_CONFIG_UI_PLUGINS))
-
-        for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
-            if field_name in self.fields:
-                get_choice_field(self, field_name).choices = [
-                    (p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins
-                ]
-
-        if "other_plugins" in self.fields:
-            get_choice_field(self, "other_plugins").choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in other_plugins]
-
-        group_specs = (
-            *PLUGIN_GROUP_DEFINITIONS,
-            ("other_plugins", "Other", "", "", "", other_plugins),
-        )
-        self.plugin_groups = [
-            {
-                "field_name": field_name,
-                "title": title,
-                "note": note,
-                "dom_id": dom_id,
-                "select_all_group": select_all_group,
-                "show_selectors": field_name in self.fields,
-                "plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config),
-            }
-            for field_name, title, note, dom_id, select_all_group, plugin_names in group_specs
-            if any(plugin in all_plugins for plugin in plugin_names)
-        ]
-
-    def _build_plugin_cards(
-        self,
-        field_name: str,
-        plugin_names: Iterable[str],
-        plugin_configs: dict[str, dict[str, Any]],
-        runtime_config: Mapping[str, Any],
-    ) -> list[dict[str, Any]]:
-        if field_name in self.fields:
-            choices = list(get_choice_field(self, field_name).choices)
-            selected_values = set(self.data.getlist(field_name)) if self.is_bound else set(get_choice_field(self, field_name).initial or [])
-        else:
-            all_plugins = get_plugins()
-            choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins]
-            selected_values = set()
-
-        cards = []
-        for index, (plugin_name, label) in enumerate(choices):
-            schema = plugin_configs.get(str(plugin_name), {})
-            properties = schema.get("properties") or {}
-            enabled_config_key = f"{str(plugin_name).upper()}_ENABLED"
-            enabled_prop_schema = properties.get(enabled_config_key)
-            if not isinstance(enabled_prop_schema, dict) or "boolean" not in _schema_types(enabled_prop_schema):
-                enabled_config_key = ""
-            config_fields = [
-                self._build_plugin_config_field(str(plugin_name), str(config_key), prop_schema, runtime_config)
-                for config_key, prop_schema in properties.items()
-                if isinstance(prop_schema, dict)
-            ]
-            cards.append(
-                {
-                    "name": str(plugin_name),
-                    "label": label,
-                    "checked": str(plugin_name) in selected_values,
-                    "checkbox_id": f"id_{field_name}_{index}",
-                    "enabled_config_key": enabled_config_key,
-                    "description": str(schema.get("description") or "").strip(),
-                    "required_plugins": [str(item) for item in schema.get("required_plugins") or []],
-                    "required_binaries_count": len(schema.get("required_binaries") or []),
-                    "config_fields": config_fields,
-                    "config_count": len(config_fields),
-                },
-            )
-        return cards
-
-    def _build_plugin_config_field(
-        self,
-        plugin_name: str,
-        config_key: str,
-        prop_schema: Mapping[str, Any],
-        runtime_config: Mapping[str, Any],
-    ) -> dict[str, Any]:
-        schema_types = _schema_types(prop_schema)
-        enum = prop_schema.get("enum")
-        input_name = _plugin_config_input_name(plugin_name, config_key)
-        current_value = runtime_config.get(config_key, prop_schema.get("default", ""))
-        if self.is_bound and input_name in self.data:
-            try:
-                current_value = _coerce_plugin_config_value(self.data.get(input_name), prop_schema)
-            except (TypeError, ValueError, json.JSONDecodeError, forms.ValidationError):
-                current_value = self.data.get(input_name)
-
-        default_value = prop_schema.get("default", "")
-        fallback_key = prop_schema.get("x-fallback")
-        default_display = f"{{{fallback_key}}}" if fallback_key else default_value
-        is_sensitive = bool(prop_schema.get("x-sensitive"))
-        input_value = "" if is_sensitive else _jsonish(current_value)
-        field_kind = "text"
-        input_type = "text"
-        options = []
-
-        if "boolean" in schema_types:
-            field_kind = "boolean"
-            input_value = "true" if bool(current_value) else "false"
-        elif isinstance(enum, list) and enum:
-            field_kind = "select"
-            options = [
-                {
-                    "value": str(option),
-                    "label": str(option),
-                    "selected": str(option) == str(current_value),
-                }
-                for option in enum
-            ]
-        elif "integer" in schema_types or "number" in schema_types:
-            field_kind = "number"
-            input_type = "number"
-        elif "array" in schema_types or "object" in schema_types:
-            field_kind = "json"
-            input_value = "" if is_sensitive else json.dumps(current_value, indent=2, sort_keys=True, default=str)
-        elif is_sensitive:
-            input_type = "password"
-
-        return {
-            "key": config_key,
-            "input_name": input_name,
-            "kind": field_kind,
-            "input_type": input_type,
-            "value": input_value,
-            "checked": bool(current_value),
-            "options": options,
-            "description": str(prop_schema.get("description") or "").strip(),
-            "default": _jsonish(default_display),
-            "current": "configured" if is_sensitive and current_value else _jsonish(current_value),
-            "current_url": self.plugin_config_binary_urls.get(config_key, "") if str(config_key).endswith("_BINARY") else "",
-            "is_sensitive": is_sensitive,
-            "minimum": prop_schema.get("minimum"),
-            "maximum": prop_schema.get("maximum"),
-            "pattern": prop_schema.get("pattern"),
-            "type_label": " / ".join(schema_types),
-        }
-
-    def clean_plugin_config_overrides(self, effective_config: Mapping[str, Any] | None = None) -> dict[str, Any]:
-        if not self.is_bound:
-            return {}
-
-        effective_config = effective_config or get_config()
-        overrides: dict[str, Any] = {}
-        sources: dict[str, str] = {}
-
-        for plugin_name, schema in discover_plugin_configs().items():
-            for config_key, prop_schema in (schema.get("properties") or {}).items():
-                if not isinstance(prop_schema, dict):
-                    continue
-
-                input_name = _plugin_config_input_name(plugin_name, config_key)
-                if input_name not in self.data:
-                    continue
-
-                raw_value: Any = self.data.get(input_name)
-                if "array" in _schema_types(prop_schema) and isinstance(prop_schema.get("enum"), list):
-                    raw_value = self.data.getlist(input_name)
-
-                if prop_schema.get("x-sensitive") and raw_value == "":
-                    continue
-
-                try:
-                    coerced_value = _coerce_plugin_config_value(raw_value, prop_schema)
-                except (TypeError, ValueError, json.JSONDecodeError) as err:
-                    self.add_error("config", forms.ValidationError(f"{config_key}: {err}"))
-                    continue
-                except forms.ValidationError as err:
-                    self.add_error("config", forms.ValidationError(f"{config_key}: {err.messages[0]}"))
-                    continue
-
-                base_value = effective_config.get(config_key, prop_schema.get("default", ""))
-                if _same_config_value(coerced_value, base_value):
-                    continue
-
-                existing_value = overrides.get(config_key)
-                if config_key in overrides and not _same_config_value(existing_value, coerced_value):
-                    self.add_error(
-                        "config",
-                        forms.ValidationError(
-                            f"{config_key} was set differently under {sources[config_key]} and {plugin_name}. Set it once in Custom config overrides.",
-                        ),
-                    )
-                    continue
-
-                overrides[config_key] = coerced_value
-                sources[config_key] = plugin_name
-
-        return overrides
-
-    def plugin_config_keys(self) -> set[str]:
-        return {
-            str(config_key)
-            for schema in discover_plugin_configs().values()
-            for config_key, prop_schema in (schema.get("properties") or {}).items()
-            if isinstance(prop_schema, dict)
-        }
-
-
-def get_plugin_config_binary_urls(runtime_config: Mapping[str, Any]) -> dict[str, str]:
-    from archivebox.config.views import get_environment_binary_url, get_installed_binary_change_url
-    from archivebox.machine.models import Binary, Machine
-
-    binary_keys = {
-        str(config_key)
-        for schema in discover_plugin_configs().values()
-        for config_key, prop_schema in (schema.get("properties") or {}).items()
-        if isinstance(prop_schema, dict) and str(config_key).endswith("_BINARY")
-    }
-    urls: dict[str, str] = {}
-    machine = Machine.current()
-    for key in binary_keys:
-        value = str(runtime_config.get(key) or "").strip()
-        if not value:
-            continue
-        name = Path(value).name if "/" in value else value
-        binary = Binary.objects.get_valid_binary(value, machine=machine)
-        if binary is None and "/" in value:
-            binary = (
-                Binary.objects.exclude(abspath="")
-                .exclude(abspath__isnull=True)
-                .filter(machine=machine, abspath=value)
-                .order_by("-modified_at")
-                .first()
-            )
-        if binary is None and name != value:
-            binary = Binary.objects.get_valid_binary(name, machine=machine)
-        urls[key] = get_installed_binary_change_url(getattr(binary, "name", name), binary) or get_environment_binary_url(name)
-    return urls
-
 
 class AddLinkForm(PluginConfigFormMixin, forms.Form):
+    allow_crawl_execution_config_fields = False
+
     # Basic fields
     url = forms.CharField(
         label="URLs",
@@ -514,11 +41,12 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             attrs={
                 "data-url-regex": URL_REGEX.pattern,
                 "placeholder": (
-                    "Enter URLs to archive, as one per line, CSV, JSON, or embedded in text "
-                    "(e.g. markdown, HTML, etc.). Examples:\n"
-                    "https://example.com\n"
-                    "https://news.ycombinator.com,https://news.google.com\n"
-                    "[ArchiveBox](https://github.com/ArchiveBox/ArchiveBox)"
+                    "\n",
+                    "Enter URL(s) to archive. Any format is ok: one per line, CSV, JSON, embedded in text, etc."
+                    "Examples:\n\n"
+                    "https://example.com\n\n"
+                    "https://news.ycombinator.com,https://news.google.com\n\n"
+                    "Or any text-based content [containing URLs](https://github.com/ArchiveBox/ArchiveBox)...",
                 ),
             },
         ),
@@ -685,7 +213,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         ),
     )
     persona = forms.ModelChoiceField(
-        label="Persona (authentication profile)",
+        label="Persona (configuration profile)",
         required=False,
         queryset=Persona.objects.none(),
         empty_label=None,
@@ -697,8 +225,8 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         initial="public",
         required=True,
     )
-    index_only = forms.BooleanField(
-        label="Index only dry run (add crawl but don't archive yet)",
+    start_paused = forms.BooleanField(
+        label="Start paused",
         initial=False,
         required=False,
     )
@@ -728,6 +256,13 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         self.fields["timeout"].initial = default_config.TIMEOUT
         self.fields["crawl_max_concurrent_snapshots"].initial = default_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
         self.fields["delete_after"].initial = default_config.DELETE_AFTER
+        self.fields["url_filters"].initial = {
+            "allowlist": "",
+            "denylist": "",
+            "same_domain_only": False,
+            "subpaths_only": False,
+            "only_new": bool(default_config.ONLY_NEW),
+        }
 
         if self.is_bound:
             selected_persona = (
@@ -780,12 +315,9 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 
     def clean_url(self):
         value = self.cleaned_data.get("url") or ""
-        valid_urls = []
-        for url in find_all_urls(value):
-            valid_urls.append(url)
-        if not valid_urls:
+        if not list(find_all_urls(value)):
             raise forms.ValidationError("Enter at least one valid URL.")
-        return "\n".join(valid_urls)
+        return value
 
     def clean_url_filters(self):
         from archivebox.crawls.models import Crawl
@@ -796,6 +328,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
             "same_domain_only": bool(value.get("same_domain_only")),
             "subpaths_only": bool(value.get("subpaths_only")),
+            "only_new": bool(value.get("only_new")),
         }
 
     def clean_max_urls(self):

archivebox/core/middleware.py

@@ -3,6 +3,7 @@ __package__ = "archivebox.core"
 import ipaddress
 import re
 from pathlib import Path
+from django.conf import settings
 from django.utils import timezone
 from django.contrib.auth.middleware import RemoteUserMiddleware
 from django.contrib.auth.models import AnonymousUser
@@ -15,15 +16,15 @@ from django.http import HttpResponseForbidden, HttpResponseNotModified
 from archivebox.config.common import get_config
 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH
-from archivebox.core.host_utils import (
+from archivebox.core.routes_util import (
     build_snapshot_url,
     build_admin_url,
     build_web_url,
     get_api_host,
     get_admin_host,
+    get_base_host,
     get_listen_host,
     get_listen_subdomain,
-    get_public_host,
     get_web_host,
     host_matches,
     is_snapshot_subdomain,
@@ -36,14 +37,32 @@ ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in"
 
 
 def _admin_login_hint_cookie_domain(config) -> str | None:
+    """Resolve the parent domain to scope the cross-subdomain login hint.
+
+    NOTE: this cookie carries only the single bit "user is logged in on
+    admin somewhere"; it MUST NOT be confused with the session cookie,
+    which stays admin-host-scoped (see core/settings.py
+    SESSION_COOKIE_DOMAIN comment — admin/web is a security boundary).
+
+    Returns the hostname portion of ``get_base_host`` (which respects
+    ``BASE_URL`` and falls back to the local-bind mapping). Strips the
+    port — cookie ``Domain=`` attributes don't include ports. Returns
+    ``None`` when subdomain routing is off, the base host is empty, or
+    the base host is an IP / bare ``localhost`` (browsers reject
+    cross-host cookies for those).
+    """
     if not config.USES_SUBDOMAIN_ROUTING:
         return None
-    listen_host, _listen_port = split_host_port(get_listen_host(config=config))
+    base_host = get_base_host(config=config)
+    if not base_host:
+        return None
+    hostname, _port = split_host_port(base_host)
+    if not hostname or hostname == "localhost":
+        return None
     try:
-        ipaddress.ip_address(listen_host)
+        ipaddress.ip_address(hostname)
     except ValueError:
-        if listen_host and listen_host != "localhost":
-            return listen_host
+        return hostname
     return None
 
 
@@ -66,6 +85,34 @@ def TimezoneMiddleware(get_response):
     return middleware
 
 
+def AdminCookieIsolationMiddleware(get_response):
+    def middleware(request):
+        response = get_response(request)
+
+        config = request.__dict__.get("archivebox_config")
+        if config is None:
+            config = get_config(resolve_plugins=False)
+            request.archivebox_config = config
+        if not config.USES_SUBDOMAIN_ROUTING:
+            return response
+
+        request_host = (request.get_host() or "").lower()
+        if host_matches(request_host, get_admin_host(config=config)):
+            return response
+
+        if host_matches(request_host, get_web_host(config=config)):
+            for cookie_name in tuple(response.cookies.keys()):
+                if cookie_name != ADMIN_LOGIN_HINT_COOKIE:
+                    response.cookies.pop(cookie_name, None)
+            return response
+
+        response.cookies.pop(settings.SESSION_COOKIE_NAME, None)
+        response.cookies.pop(settings.CSRF_COOKIE_NAME, None)
+        return response
+
+    return middleware
+
+
 def CacheControlMiddleware(get_response):
     snapshot_path_re = re.compile(r"^/[^/]+/\\d{8}/[^/]+/[0-9a-fA-F-]{8,36}/")
     static_cache_key = (get_COMMIT_HASH() or VERSION or "dev").strip()
@@ -100,7 +147,7 @@ def CacheControlMiddleware(get_response):
 
         if "/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path):
             if not response.get("Cache-Control"):
-                config = getattr(request, "archivebox_config", None)
+                config = request.__dict__.get("archivebox_config")
                 if config is None:
                     config = get_config(resolve_plugins=False)
                     request.archivebox_config = config
@@ -117,7 +164,7 @@ def ServerSecurityModeMiddleware(get_response):
     allowed_methods = {"GET", "HEAD", "OPTIONS"}
 
     def middleware(request):
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
         if config is None:
             config = get_config(resolve_plugins=False)
             request.archivebox_config = config
@@ -150,14 +197,13 @@ def HostRoutingMiddleware(get_response):
             return get_response(request)
 
         request_host = (request.get_host() or "").lower()
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
         if config is None:
             config = get_config(resolve_plugins=False)
             request.archivebox_config = config
         admin_host = get_admin_host(config=config)
         web_host = get_web_host(config=config)
         api_host = get_api_host(config=config)
-        public_host = get_public_host(config=config)
         listen_host = get_listen_host(config=config)
         subdomain = get_listen_subdomain(request_host, config=config)
 
@@ -166,18 +212,34 @@ def HostRoutingMiddleware(get_response):
         if request.path.startswith("/static/") or request.path in {"/favicon.ico", "/robots.txt"}:
             return get_response(request)
 
-        if config.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host):
+        if config.USES_SUBDOMAIN_ROUTING and config.BASE_URL and not host_matches(request_host, admin_host):
+            add_should_redirect = not config.PUBLIC_ADD_VIEW and (request.path == "/add" or request.path.startswith("/add/"))
             if (
                 request.path == "/admin"
                 or request.path.startswith("/admin/")
                 or request.path == "/accounts"
                 or request.path.startswith("/accounts/")
+                or add_should_redirect
             ):
                 target = build_admin_url(request.path, request=request)
                 if request.META.get("QUERY_STRING"):
                     target = f"{target}?{request.META['QUERY_STRING']}"
                 return redirect(target)
 
+        if subdomain and is_snapshot_subdomain(subdomain):
+            view = SnapshotHostView.as_view()
+            return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
+
+        # In subdomain mode with no explicit BASE_URL we can't safely emit
+        # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
+        # request's own Host (via the request-host fallback in get_base_url),
+        # so prepending ``admin.`` to whatever the client sent produces a
+        # redirect chain of ``admin.admin.admin.<host>``. Pass the request
+        # through; the misconfig banner on the rendered page tells the user
+        # to pin BASE_URL so the redirects can resume.
+        if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
+            return get_response(request)
+
         if not config.USES_SUBDOMAIN_ROUTING:
             if host_matches(request_host, listen_host):
                 return get_response(request)
@@ -243,16 +305,7 @@ def HostRoutingMiddleware(get_response):
             request._cached_user = request.user
             return get_response(request)
 
-        if host_matches(request_host, public_host):
-            if request.COOKIES.get(ADMIN_LOGIN_HINT_COOKIE) == "1" and (request.path == "/public" or request.path.startswith("/public/")):
-                target = build_admin_url("/admin/core/snapshot/", request=request)
-                return redirect(target)
-            return get_response(request)
-
         if subdomain:
-            if is_snapshot_subdomain(subdomain):
-                view = SnapshotHostView.as_view()
-                return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
             view = OriginalDomainHostView.as_view()
             return view(request, domain=subdomain, path=request.path.lstrip("/"))
 
@@ -262,7 +315,13 @@ def HostRoutingMiddleware(get_response):
                 target = f"{target}?{request.META['QUERY_STRING']}"
             return redirect(target)
 
-        if admin_host or web_host:
+        if (admin_host or web_host) and config.BASE_URL:
+            # Only force a canonical-host redirect when BASE_URL was set
+            # explicitly. If BASE_URL is empty (e.g. 0.7.3 → 0.9.0 upgrade
+            # where the user has CSRF_TRUSTED_ORIGINS but never set BASE_URL),
+            # the subdomain we'd redirect to may not actually resolve in the
+            # user's reverse proxy — serve the request as-is instead and let
+            # the misconfig banner surface the problem in the page.
             target = build_web_url(request.path, request=request)
             if target:
                 if request.META.get("QUERY_STRING"):
@@ -278,7 +337,7 @@ class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
     header = "HTTP_REMOTE_USER"
 
     def process_request(self, request):
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
         if config is None:
             config = get_config(resolve_plugins=False)
             request.archivebox_config = config

archivebox/core/migrations/0045_archiveresult_unique_hook.py

@@ -1,12 +1,40 @@
 from django.db import migrations, models
 
 
+def deduplicate_archiveresults_per_hook(apps, schema_editor):
+    """Drop duplicate ArchiveResult rows per (snapshot, plugin, hook_name).
+
+    Real long-lived collections (cabbage's demo, beta-tester DBs) accumulated
+    multiple rows per hook over the dev rc chain. The next operation adds a
+    UniqueConstraint on that tuple; without this cleanup pass the constraint
+    fails with ``UNIQUE constraint failed`` mid-migration and bricks startup.
+    Keep the row with the highest id (most recent) for each tuple.
+    """
+    ArchiveResult = apps.get_model("core", "ArchiveResult")
+    duplicate_groups = (
+        ArchiveResult.objects.values("snapshot_id", "plugin", "hook_name").annotate(count=models.Count("id")).filter(count__gt=1)
+    )
+    for group in duplicate_groups.iterator(chunk_size=200):
+        lookup = {
+            "snapshot_id": group["snapshot_id"],
+            "plugin": group["plugin"],
+            "hook_name": group["hook_name"],
+        }
+        keep = ArchiveResult.objects.filter(**lookup).order_by("-id").values_list("id", flat=True).first()
+        if keep is not None:
+            ArchiveResult.objects.filter(**lookup).exclude(id=keep).delete()
+
+
 class Migration(migrations.Migration):
     dependencies = [
         ("core", "0044_alter_archiveresult_status_alter_snapshot_status"),
     ]
 
     operations = [
+        migrations.RunPython(
+            deduplicate_archiveresults_per_hook,
+            reverse_code=migrations.RunPython.noop,
+        ),
         migrations.AddConstraint(
             model_name="archiveresult",
             constraint=models.UniqueConstraint(

archivebox/core/migrations/0046_repair_snapshot_permissions.py

@@ -0,0 +1,43 @@
+from django.db import migrations
+
+
+def _repair_snapshot_permissions(apps, schema_editor):
+    """Backfill ``core_snapshot.permissions`` if a prior squash-skewed
+    ``django_migrations`` row claimed ``0041_snapshot_permissions`` was
+    applied but the column never actually landed on the table.
+
+    Beta-tester / cabbage-style DBs upgraded incrementally through the
+    0.8.x → 0.9.x rc chain have a 0041_snapshot_permissions entry with a
+    different historical effect (the name was reused across squashes),
+    so the runtime model's ``snapshot.permissions`` ``GeneratedField`` has
+    no underlying column. Fresh installs added the column via 0041 and
+    this is a no-op.
+    """
+    cursor = schema_editor.connection.cursor()
+    # ``table_xinfo`` lists STORED/VIRTUAL generated columns; ``table_info``
+    # silently drops them, so a prior 0041 that landed the STORED column
+    # would otherwise look absent here and we'd ALTER TABLE -> duplicate.
+    cursor.execute("PRAGMA table_xinfo(core_snapshot)")
+    existing_cols = {row[1] for row in cursor.fetchall()}
+    if "permissions" in existing_cols:
+        return
+    # See crawls/0016 — SQLite ALTER TABLE only allows VIRTUAL generated
+    # columns; STORED would error with "cannot add a STORED column". The
+    # runtime model's ``db_persist=True`` is only honored for fresh installs
+    # where 0041 added the column during initial table creation.
+    cursor.execute(
+        "ALTER TABLE core_snapshot ADD COLUMN permissions varchar(16) GENERATED ALWAYS AS (json_extract(config, '$.PERMISSIONS')) VIRTUAL",
+    )
+    cursor.execute(
+        "CREATE INDEX IF NOT EXISTS core_snapshot_permissions_idx ON core_snapshot (permissions)",
+    )
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0045_archiveresult_unique_hook"),
+    ]
+
+    operations = [
+        migrations.RunPython(_repair_snapshot_permissions, migrations.RunPython.noop),
+    ]

archivebox/core/migrations/0047_archiveresult_status_snapshot_index.py

@@ -0,0 +1,28 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0046_repair_snapshot_permissions"),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunSQL(
+                    sql="CREATE INDEX IF NOT EXISTS archiveresult_status_snap_idx ON core_archiveresult (status, snapshot_id)",
+                    reverse_sql="DROP INDEX IF EXISTS archiveresult_status_snap_idx",
+                ),
+            ],
+            state_operations=[
+                migrations.AddIndex(
+                    model_name="archiveresult",
+                    index=models.Index(fields=["status", "snapshot"], name="archiveresult_status_snap_idx"),
+                ),
+            ],
+        ),
+        migrations.RunSQL(
+            sql="ANALYZE core_archiveresult",
+            reverse_sql=migrations.RunSQL.noop,
+        ),
+    ]

archivebox/core/migrations/0048_remove_archiveresult_config.py

@@ -0,0 +1,14 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0047_archiveresult_status_snapshot_index"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="archiveresult",
+            name="config",
+        ),
+    ]

archivebox/core/migrations/0049_alter_snapshot_url.py

@@ -0,0 +1,17 @@
+# Generated by Django 6.0.6 on 2026-06-04 22:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0048_remove_archiveresult_config"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="url",
+            field=models.TextField(db_index=True),
+        ),
+    ]

archivebox/core/migrations/0050_snapshot_permissions_not_null.py

@@ -0,0 +1,69 @@
+import django.db.models.fields.json
+import django.db.models.functions.comparison
+from django.db import migrations, models
+from django.db.models import Q
+
+
+VALID_PERMISSIONS = {"public", "unlisted", "private"}
+BATCH_SIZE = 1000
+
+
+def normalize_permissions(value, default="private"):
+    value = str(value or "").strip().lower()
+    return value if value in VALID_PERMISSIONS else default
+
+
+def hydrate_snapshot_permissions(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    db_alias = schema_editor.connection.alias
+    missing_permissions = Q(permissions__isnull=True) | (Q(permissions__isnull=False) & ~Q(permissions__in=VALID_PERMISSIONS))
+    batch = []
+
+    snapshots = (
+        Snapshot.objects.using(db_alias)
+        .filter(missing_permissions)
+        .select_related("crawl")
+        .only("id", "config", "crawl__permissions")
+        .iterator(chunk_size=BATCH_SIZE)
+    )
+    for snapshot in snapshots:
+        config = dict(snapshot.config or {})
+        config["PERMISSIONS"] = normalize_permissions(snapshot.crawl.permissions)
+        snapshot.config = config
+        batch.append(snapshot)
+        if len(batch) >= BATCH_SIZE:
+            Snapshot.objects.using(db_alias).bulk_update(batch, ["config"], batch_size=BATCH_SIZE)
+            batch.clear()
+
+    if batch:
+        Snapshot.objects.using(db_alias).bulk_update(batch, ["config"], batch_size=BATCH_SIZE)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("crawls", "0016_hydrate_crawl_permissions"),
+        ("core", "0049_alter_snapshot_url"),
+    ]
+
+    operations = [
+        migrations.RunPython(hydrate_snapshot_permissions, migrations.RunPython.noop),
+        migrations.RemoveField(
+            model_name="snapshot",
+            name="permissions",
+        ),
+        migrations.AddField(
+            model_name="snapshot",
+            name="permissions",
+            field=models.GeneratedField(
+                db_index=True,
+                db_persist=True,
+                editable=False,
+                expression=django.db.models.functions.comparison.Coalesce(
+                    django.db.models.fields.json.KeyTextTransform("PERMISSIONS", "config"),
+                    models.Value("private"),
+                    output_field=models.CharField(max_length=16),
+                ),
+                output_field=models.CharField(max_length=16),
+            ),
+        ),
+    ]

archivebox/core/permissions.py

@@ -1,10 +1,8 @@
 from __future__ import annotations
 
-from django.db.models import Q, QuerySet
+from django.db.models import QuerySet
 from django.http import HttpRequest
 
-from archivebox.config.common import get_config
-
 PERMISSIONS_PUBLIC = "public"
 PERMISSIONS_UNLISTED = "unlisted"
 PERMISSIONS_PRIVATE = "private"
@@ -13,6 +11,19 @@ PERMISSIONS_CHOICES = (
     (PERMISSIONS_UNLISTED, "Unlisted"),
     (PERMISSIONS_PRIVATE, "Private"),
 )
+PERMISSIONS_VALUES = {value for value, _label in PERMISSIONS_CHOICES}
+PERMISSIONS_META = {
+    PERMISSIONS_PUBLIC: ("👥", "Public", "#047857", "#d1fae5"),
+    PERMISSIONS_UNLISTED: ("🔗", "Unlisted", "#1d4ed8", "#dbeafe"),
+    PERMISSIONS_PRIVATE: ("🔒", "Private", "#991b1b", "#fee2e2"),
+}
+
+
+def normalize_permissions(permissions: object, *, default: str = PERMISSIONS_PRIVATE) -> str:
+    permissions = str(permissions or "").strip().lower()
+    if permissions not in PERMISSIONS_VALUES:
+        return default
+    return permissions
 
 
 def is_admin_user(request: HttpRequest) -> bool:
@@ -21,10 +32,7 @@ def is_admin_user(request: HttpRequest) -> bool:
 
 
 def get_snapshot_permissions(snapshot) -> str:
-    try:
-        return str(get_config(snapshot=snapshot, resolve_plugins=False).PERMISSIONS).strip().lower()
-    except Exception:
-        return PERMISSIONS_PRIVATE
+    return normalize_permissions(snapshot.permissions)
 
 
 def can_view_snapshot(request: HttpRequest, snapshot) -> bool:
@@ -35,9 +43,7 @@ def can_view_snapshot(request: HttpRequest, snapshot) -> bool:
 def _persona_ids_for_permissions(allowed_permissions: set[str]) -> list[str]:
     from archivebox.personas.models import Persona
 
-    fallback_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower()
-    personas = Persona.objects.only("id", "config")
-    return [str(persona.id) for persona in personas if (persona.permissions or fallback_permissions) in allowed_permissions]
+    return [str(persona_id) for persona_id in Persona.objects.filter(permissions__in=allowed_permissions).values_list("id", flat=True)]
 
 
 def filter_personas_by_permissions(queryset: QuerySet, allowed_permissions: set[str]) -> QuerySet:
@@ -45,28 +51,9 @@ def filter_personas_by_permissions(queryset: QuerySet, allowed_permissions: set[
 
 
 def filter_snapshots_by_permissions(queryset: QuerySet, *, direct: bool = False, allowed_permissions: set[str] | None = None) -> QuerySet:
-    from archivebox.crawls.models import Crawl
-    from archivebox.personas.models import Persona
-
     allowed_permissions = allowed_permissions or ({PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED} if direct else {PERMISSIONS_PUBLIC})
-    fallback_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower()
-    has_overrides = (
-        queryset.model.objects.filter(permissions__gt="").exists()
-        or Crawl.objects.filter(permissions__gt="").exists()
-        or Persona.objects.filter(permissions__gt="").exists()
-    )
-    if not has_overrides:
-        return queryset if fallback_permissions in allowed_permissions else queryset.none()
-
-    allowed_persona_ids = _persona_ids_for_permissions(allowed_permissions)
-    valid_persona_ids = [str(persona_id) for persona_id in Persona.objects.values_list("id", flat=True)]
-    fallback_query = Q(crawl__persona_id__in=allowed_persona_ids)
-    if fallback_permissions in allowed_permissions:
-        fallback_query |= Q(crawl__persona_id__isnull=True) | ~Q(crawl__persona_id__in=valid_persona_ids)
-    inherited_query = Q(crawl__permissions__in=sorted(allowed_permissions)) | (Q(crawl__permissions__isnull=True) & fallback_query)
-    return queryset.filter(
-        Q(permissions__in=sorted(allowed_permissions)) | (Q(permissions__isnull=True) & inherited_query),
-    )
+    allowed = sorted(allowed_permissions)
+    return queryset.filter(permissions__in=allowed)
 
 
 def public_snapshots_queryset(queryset: QuerySet) -> QuerySet:

archivebox/core/preview_util.py

@@ -0,0 +1,4 @@
+__package__ = "archivebox.core"
+
+
+EXTENSION_SCREENSHOT_PLUGIN = "chrome_extension_screenshot"

archivebox/core/recovery_util.py

@@ -1,28 +1,40 @@
 from __future__ import annotations
 
+from pathlib import Path
+
 from django.utils import timezone
 from rich.console import Console
 
 
+def _is_signal_interrupted_exit(exit_code: int | None) -> bool:
+    return exit_code is not None and (exit_code < 0 or exit_code >= 128)
+
+
 def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int]:
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.services.archive_result_service import _collect_output_metadata
     from archivebox.machine.models import Process
+    from django.core.exceptions import ValidationError
     from django.db.models import Exists, OuterRef, Q, Subquery, Value
     from django.db.models.functions import Coalesce
 
     now = timezone.now()
     recovery_console = Console(stderr=True, highlight=False, soft_wrap=True)
     cleaned = {
-        "stale_processes": Process.cleanup_stale_running(),
-        "orphaned_processes": Process.cleanup_orphaned_workers(),
-        "orphaned_chrome": Process.cleanup_orphaned_chrome() if include_chrome else 0,
-        "queued_crawls_unlocked": 0,
-        "unlocked_snapshots": 0,
-        "queued_snapshots_unlocked": 0,
-        "requeued_archiveresults": 0,
-        "queued_snapshot_maintenance_scheduled": 0,
-        "unlocked_crawls": 0,
+        "processes_stale_running": Process.cleanup_stale_running(),
+        "processes_orphaned_workers": Process.cleanup_orphaned_workers(),
+        "chrome_processes_orphaned": Process.cleanup_orphaned_chrome() if include_chrome else 0,
+        "crawls_queued_without_retry_at": 0,
+        "snapshots_queued_without_retry_at": 0,
+        "archiveresults_backoff": 0,
+        "snapshots_queued_plugin_rows_waiting_on_stale_lease": 0,
+        "archiveresults_started_without_running_process": 0,
+        "archiveresults_missing_for_orphaned_hook_processes": 0,
+        "snapshots_started_without_running_results": 0,
+        "crawls_started_with_due_snapshots": 0,
+        "crawls_started_waiting_on_future_snapshots": 0,
+        "crawls_started_without_active_snapshots": 0,
     }
 
     running_archiveresults = ArchiveResult.objects.filter(
@@ -32,7 +44,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
     )
     active_child_snapshots = Snapshot.objects.filter(
         crawl_id=OuterRef("pk"),
-        status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.PAUSED],
+        status__in=Snapshot.OPEN_STATES,
     )
     due_child_snapshots = active_child_snapshots.exclude(status=Snapshot.StatusChoices.PAUSED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__lte=now),
@@ -43,45 +55,130 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
 
     # Broken lock repair: QUEUED rows with retry_at=NULL are invisible to the
     # queue. Set only the scheduling field so the runner owns the next tick.
-    cleaned["queued_crawls_unlocked"] = Crawl.objects.filter(
+    cleaned["crawls_queued_without_retry_at"] = Crawl.objects.filter(
         status=Crawl.StatusChoices.QUEUED,
         retry_at__isnull=True,
     ).update(retry_at=now, modified_at=now)
-    cleaned["queued_snapshots_unlocked"] = Snapshot.objects.filter(
+    cleaned["snapshots_queued_without_retry_at"] = Snapshot.objects.filter(
         status=Snapshot.StatusChoices.QUEUED,
         retry_at__isnull=True,
+        crawl__status__in=Crawl.RUNNABLE_STATES,
     ).update(retry_at=now, modified_at=now)
-    # ArchiveResult has no retry_at scheduler; BACKOFF is a legacy/impossible
-    # persisted state here, so move it back to QUEUED for the snapshot runner.
-    cleaned["requeued_archiveresults"] = ArchiveResult.objects.filter(
-        status=ArchiveResult.StatusChoices.BACKOFF,
-    ).update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now)
+    backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF)
+    orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED).exclude(
+        process__status=Process.StatusChoices.RUNNING,
+    )
+    # ArchiveResult has no retry_at scheduler. Wake only the parent Snapshots
+    # for result rows we are about to repair, then requeue those exact rows.
+    # Using subqueries keeps million-row recovery in SQLite instead of building
+    # Python ID lists or scanning all sealed snapshots.
+    Snapshot.objects.filter(
+        id__in=backoff_results.values("snapshot_id"),
+        status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
+        retry_at__isnull=True,
+    ).update(retry_at=now, modified_at=now)
+    Snapshot.objects.filter(
+        id__in=orphaned_results.values("snapshot_id"),
+        status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
+        retry_at__isnull=True,
+    ).update(retry_at=now, modified_at=now)
+    cleaned["archiveresults_backoff"] = backoff_results.update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now)
+    # Targeted plugin rows on final/paused Snapshots are scheduled through the
+    # parent Snapshot.retry_at. retry_at=NULL is the normal idle marker for a
+    # sealed Snapshot and must not be interpreted as queued work just because
+    # old/synthetic ArchiveResult rows exist. If takeover kills the runner
+    # after it leases the Snapshot but before queued ArchiveResult rows finish,
+    # the rows remain QUEUED while retry_at sits in the future. Recovery runs
+    # only after this runner has won the single-runner gate, so it can safely
+    # unlock those stale plugin leases for immediate processing instead of
+    # waiting out the previous owner's full lock timeout.
+    queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED)
+    cleaned["snapshots_queued_plugin_rows_waiting_on_stale_lease"] = (
+        Snapshot.objects.filter(
+            id__in=queued_plugin_results.values("snapshot_id"),
+            status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
+        )
+        .filter(retry_at__gt=now)
+        .update(retry_at=now, modified_at=now)
+    )
     # Impossible state repair: STARTED ArchiveResults without a live Process
     # have no owner left to emit completion. Requeue only the result row; the
     # snapshot/crawl schedulers will pick up normal retry processing.
-    cleaned["requeued_archiveresults"] += (
-        ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED)
-        .exclude(process__status=Process.StatusChoices.RUNNING)
-        .update(status=ArchiveResult.StatusChoices.QUEUED, process=None, modified_at=now)
-    )
-    # SEALED snapshots can legitimately carry targeted maintenance work
-    # (`update --index-only`, filesystem migrations). Status stays SEALED; this
-    # only makes existing queued ArchiveResult rows visible to the sealed
-    # snapshot branch of the runner.
-    queued_maintenance_snapshot_ids = tuple(
-        Snapshot.objects.filter(
-            status=Snapshot.StatusChoices.SEALED,
-            retry_at__isnull=True,
-            archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
-        )
-        .order_by("modified_at", "id")
-        .values_list("id", flat=True)[:1000],
-    )
-    cleaned["queued_snapshot_maintenance_scheduled"] = Snapshot.objects.filter(id__in=queued_maintenance_snapshot_ids).update(
-        retry_at=now,
+    cleaned["archiveresults_started_without_running_process"] = orphaned_results.update(
+        status=ArchiveResult.StatusChoices.QUEUED,
+        process=None,
         modified_at=now,
     )
-
+    orphaned_hook_processes = Process.objects.filter(
+        process_type=Process.TypeChoices.HOOK,
+        archiveresult__isnull=True,
+    ).exclude(status=Process.StatusChoices.RUNNING)
+    for process in orphaned_hook_processes.only("id", "pwd", "cmd", "process_type", "status"):
+        hook_script_name = process.hook_script_name
+        if not hook_script_name or not process.pwd:
+            continue
+        plugin_dir = Path(process.pwd)
+        try:
+            # Old or synthetic hook Process rows can point at arbitrary paths.
+            # Only paths whose parent directory is a valid Snapshot id can be
+            # reconstructed into ArchiveResult rows.
+            snapshot = Snapshot.objects.filter(id=plugin_dir.parent.name).first()
+        except ValidationError:
+            continue
+        if snapshot is None:
+            continue
+        result, created = ArchiveResult.objects.get_or_create(
+            snapshot=snapshot,
+            plugin=plugin_dir.name,
+            hook_name=Path(hook_script_name).stem,
+            defaults={
+                "status": ArchiveResult.StatusChoices.QUEUED,
+            },
+        )
+        if result.status == ArchiveResult.StatusChoices.QUEUED:
+            requeue_snapshot = False
+            # A runner can die after the hook Process exits but before the
+            # ProcessCompletedEvent projector links/finalizes ArchiveResult.
+            # Reconstruct only that exact hook row from the durable Process row.
+            output_files, output_size, output_mimetypes = _collect_output_metadata(plugin_dir)
+            result.process = process
+            if _is_signal_interrupted_exit(process.exit_code):
+                # The owning runner died or was asked to stop while the hook was
+                # still active. Keep the work item queued so takeover retries the
+                # same hook; treating an unknown signal exit as success would
+                # silently skip unfinished side effects.
+                result.output_files = {}
+                result.output_size = 0
+                result.output_mimetypes = ""
+                result.output_str = ""
+                result.status = ArchiveResult.StatusChoices.QUEUED
+                requeue_snapshot = True
+            else:
+                result.output_files = output_files
+                result.output_size = output_size
+                result.output_mimetypes = output_mimetypes
+                result.output_str = process.stderr if process.exit_code not in (0, None) else ""
+                result.status = (
+                    ArchiveResult.StatusChoices.FAILED
+                    if process.exit_code not in (0, None)
+                    else (ArchiveResult.StatusChoices.SUCCEEDED if output_files else ArchiveResult.StatusChoices.NORESULTS)
+                )
+            result.save(
+                update_fields=[
+                    "process",
+                    "output_files",
+                    "output_size",
+                    "output_mimetypes",
+                    "output_str",
+                    "status",
+                    "modified_at",
+                ],
+            )
+            if requeue_snapshot:
+                Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
+        if created:
+            cleaned["archiveresults_missing_for_orphaned_hook_processes"] += 1
+            Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
     started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__gt=now),
     )
@@ -93,7 +190,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
     # previous runner's full lease before the new runner can resume them.
     # We only unlock scheduling; normal Snapshot runner code owns the next
     # transition and side effects.
-    cleaned["unlocked_snapshots"] = (
+    cleaned["snapshots_started_without_running_results"] = (
         started_snapshots.annotate(has_running_results=Exists(running_archiveresults))
         .filter(has_running_results=False)
         .update(
@@ -111,40 +208,69 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
     )
 
     due_started_crawls = recoverable_started_crawls.annotate(has_due_child=Exists(due_child_snapshots)).filter(has_due_child=True)
-    cleaned["unlocked_crawls"] = due_started_crawls.update(retry_at=now, modified_at=now)
+    cleaned["crawls_started_with_due_snapshots"] = due_started_crawls.update(retry_at=now, modified_at=now)
     future_started_crawls = recoverable_started_crawls.annotate(
         has_active_child=Exists(active_child_snapshots),
         has_due_child=Exists(due_child_snapshots),
         next_child_retry=next_future_child_retry,
     ).filter(has_active_child=True, has_due_child=False)
-    cleaned["unlocked_crawls"] += future_started_crawls.update(retry_at=Coalesce("next_child_retry", Value(now)), modified_at=now)
+    cleaned["crawls_started_waiting_on_future_snapshots"] = future_started_crawls.update(
+        retry_at=Coalesce("next_child_retry", Value(now)),
+        modified_at=now,
+    )
     finished_started_crawls = recoverable_started_crawls.annotate(has_active_child=Exists(active_child_snapshots)).filter(
         has_active_child=False,
     )
-    cleaned["unlocked_crawls"] += finished_started_crawls.update(retry_at=now, modified_at=now)
+    cleaned["crawls_started_without_active_snapshots"] = finished_started_crawls.update(retry_at=now, modified_at=now)
 
-    warning_recoveries = {
-        "stale_processes": "marked stale running Process row(s) exited",
-        "orphaned_processes": "marked orphaned worker/hook Process row(s) exited",
-        "orphaned_chrome": "terminated orphaned Chrome process(es)",
-        "unlocked_snapshots": "unlocked started Snapshot row(s) whose owner process was gone",
-        "queued_snapshot_maintenance_scheduled": "scheduled sealed Snapshot row(s) with queued maintenance ArchiveResults",
+    repair_messages = {
+        "processes_stale_running": (
+            "Closing {count} interrupted process(es) "
+            "(ArchiveBox may have been interrupted before it was able to record that they stopped; any affected work can now be retried)."
+        ),
+        "processes_orphaned_workers": (
+            "Closing {count} interrupted extractor process(es) "
+            "(ArchiveBox may have been interrupted before it was able to record their result; affected extractor results can now be retried)."
+        ),
+        "chrome_processes_orphaned": (
+            "Stopping {count} leftover browser process(es) "
+            "(ArchiveBox may have been interrupted before it was able to close them; this frees browser resources and avoids duplicate browser sessions)."
+        ),
+        "crawls_queued_without_retry_at": (
+            "Starting {count} Crawl(s) that were queued but never started "
+            "(ArchiveBox may have been interrupted before it was able to begin archiving them)."
+        ),
+        "snapshots_queued_without_retry_at": (
+            "Starting {count} Snapshot(s) that were queued but never started "
+            "(ArchiveBox may have been interrupted before it was able to archive those URLs)."
+        ),
+        "archiveresults_backoff": (
+            "Retrying {count} extractor result(s) that were waiting to retry "
+            "(ArchiveBox may have been interrupted before it was able to try them again; affected outputs will be retried)."
+        ),
+        "archiveresults_started_without_running_process": (
+            "Retrying {count} extractor result(s) that were interrupted before finishing "
+            "(ArchiveBox may have been interrupted before it was able to save their final status; partial files will be overwritten with fresh results upon retry)."
+        ),
+        "snapshots_started_without_running_results": (
+            "Resuming {count} Snapshot(s) that were interrupted before finishing "
+            "(ArchiveBox may have been interrupted before it was able to finish archiving them; missing outputs will be retried)."
+        ),
+        "crawls_started_with_due_snapshots": (
+            "Resuming {count} Crawl(s) with pending URLs ready to archive "
+            "(ArchiveBox may have been interrupted before it was able to archive the remaining URLs; pending URLs will continue)."
+        ),
+        "crawls_started_waiting_on_future_snapshots": (
+            "Resuming {count} Crawl(s) with URLs waiting for a later retry "
+            "(ArchiveBox may have been interrupted before it was able to retry delayed URLs; they will retry later)."
+        ),
+        "crawls_started_without_active_snapshots": (
+            "Finalizing {count} Crawl(s) that finished URL processing but were not closed cleanly "
+            "(ArchiveBox may have been interrupted before it was able to save the final crawl status; archived data is not changed)."
+        ),
     }
-    scheduler_repairs = {
-        "queued_crawls_unlocked": "made queued Crawl row(s) with retry_at=NULL visible to the runner",
-        "queued_snapshots_unlocked": "made queued Snapshot row(s) with retry_at=NULL visible to the runner",
-        "requeued_archiveresults": "requeued ArchiveResult row(s) whose owner process was gone",
-    }
-    for key, message in warning_recoveries.items():
+    for key, message in repair_messages.items():
         if cleaned[key]:
-            recovery_console.print(f"[yellow]⚠️ Orchestrator recovery: {cleaned[key]} {message}.[/yellow]")
-    if cleaned["unlocked_crawls"]:
-        recovery_console.print(
-            f"[yellow]⚠️ Repairing: Rescheduled {cleaned['unlocked_crawls']} Crawl row(s) "
-            "that were left unfinished by a previous runner[/yellow]",
-        )
-    for key, message in scheduler_repairs.items():
-        if cleaned[key]:
-            recovery_console.print(f"[yellow]⚠️ Orchestrator scheduler repair: {cleaned[key]} {message}.[/yellow]")
+            recovery_console.print(f"[yellow]⚠️ Repairing: {message.format(count=cleaned[key])}[/yellow]")
 
     return cleaned

archivebox/core/routes_util.py

@@ -9,6 +9,7 @@ from archivebox.config.common import get_config
 
 _SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
 _SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P<suffix>[0-9a-fA-F]{12})$")
+_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api")
 
 
 def split_host_port(host: str) -> tuple[str, str | None]:
@@ -29,13 +30,63 @@ def _normalize_base_url(value: str | None) -> str:
     parsed = urlparse(base)
     if not parsed.netloc:
         return ""
-    return f"{parsed.scheme}://{parsed.netloc}"
+    # Accept ``*.<host>`` as a synonym for ``<host>`` so users can paste the
+    # wildcard-friendly form (e.g. from the banner suggestion) without it
+    # leaking ``*.`` into every downstream URL. Subdomain routing already
+    # prepends the appropriate role label (admin/web/api/snap-*) at build
+    # time, so the bare base host is what we want to store.
+    netloc = parsed.netloc
+    while netloc.startswith("*."):
+        netloc = netloc[2:]
+    if not netloc:
+        return ""
+    return f"{parsed.scheme}://{netloc}"
 
 
 def normalize_base_url(value: str | None) -> str:
     return _normalize_base_url(value)
 
 
+def _csrf_trusted_origins(config) -> list[str]:
+    raw = (config.CSRF_TRUSTED_ORIGINS or "").strip()
+    if not raw:
+        return []
+    seen: list[str] = []
+    for entry in raw.split(","):
+        normalized = _normalize_base_url(entry.strip())
+        if normalized and normalized not in seen:
+            seen.append(normalized)
+    return seen
+
+
+def _allowed_hosts(config) -> set[str]:
+    raw = (config.ALLOWED_HOSTS or "").strip()
+    if not raw:
+        return set()
+    return {entry.strip().lower() for entry in raw.split(",") if entry.strip() and entry.strip() != "*"}
+
+
+def derive_base_url_from_csrf(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    """Pick a single CSRF_TRUSTED_ORIGINS entry to act as the implicit BASE_URL.
+
+    0.7.3 → 0.9.0 upgrade path: any reverse-proxied 0.7.3 deployment already
+    had ``CSRF_TRUSTED_ORIGINS=https://archive.example.com`` set (required for
+    admin login to work). On upgrade, ``BASE_URL`` is the new knob — but it
+    defaults to empty, and falling through to ``BIND_ADDR`` produces an
+    unreachable URL like ``http://0.0.0.0:8000``. If the user has exactly one
+    CSRF origin we treat it as the implicit BASE_URL so links/redirects keep
+    pointing at the public hostname they already configured.
+
+    Returns ``""`` when the inference is ambiguous (multiple origins) or
+    impossible (none set) so callers fall through to their next strategy.
+    """
+    config = config or get_config(**config_kwargs)
+    origins = _csrf_trusted_origins(config)
+    if len(origins) == 1:
+        return origins[0]
+    return ""
+
+
 def get_listen_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     return (config.BIND_ADDR or "").strip()
@@ -50,10 +101,48 @@ def _with_port(host: str, port: str | None) -> str:
     return f"{host}:{port}" if port else host
 
 
+def strip_role_subdomain(host: str) -> str:
+    """Strip leading ``admin.`` / ``web.`` / ``api.`` / ``snap-*.``
+    labels from a host (preserving the port). Strips repeatedly so an
+    already-compounded host like ``snap-X.snap-X.<base>`` reduces all the
+    way down to ``<base>``.
+
+    Used when we want to recover the canonical base host from a request that
+    arrived on a role subdomain — otherwise builders that prepend their own
+    role label (e.g. ``snap-X.``) compound onto the existing prefix and you
+    get ``snap-X.snap-X.snap-X.<base>`` on every click.
+    """
+    if not host:
+        return ""
+    hostname, port = split_host_port(host)
+    while hostname and "." in hostname:
+        head, _sep, rest = hostname.partition(".")
+        if head in _ROLE_SUBDOMAIN_LABELS or _SNAPSHOT_SUBDOMAIN_RE.match(head):
+            hostname = rest
+            continue
+        break
+    return _with_port(hostname, port)
+
+
 def _is_local_bind_host(host: str) -> bool:
     return host in {"", "0.0.0.0", "::", "127.0.0.1", "::1", "localhost"}
 
 
+def canonical_base_host_for_request(request_host: str) -> str:
+    """Strip role subdomains and remap loopback hostnames to ``archivebox.localhost``.
+
+    Used by the banner suggestion and the in-browser pin endpoint: when the
+    user is hitting the server on raw ``localhost:9292`` or ``127.0.0.1:9292``
+    we want to suggest the wildcard-friendly ``archivebox.localhost`` family
+    instead, so the eventual pinned ``BASE_URL`` plays nicely with subdomain
+    routing without forcing the user to add a /etc/hosts entry.
+    """
+    hostname, port = split_host_port(strip_role_subdomain(request_host or ""))
+    if _is_local_bind_host(hostname):
+        hostname = "archivebox.localhost"
+    return _with_port(hostname, port)
+
+
 def _root_host_from_listen(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     listen_host, listen_port = get_listen_parts(config=config)
@@ -67,6 +156,13 @@ def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kw
     if override:
         return override
 
+    # A) Implicit BASE_URL from a single CSRF_TRUSTED_ORIGINS entry. Catches
+    # 0.7.3 → 0.9.0 upgrades where users already set CSRF_TRUSTED_ORIGINS
+    # for their reverse-proxy login but never set BASE_URL.
+    csrf_derived = derive_base_url_from_csrf(config)
+    if csrf_derived:
+        return csrf_derived
+
     scheme = request.scheme if request else "http"
     if request:
         req_host, req_port = split_host_port(request.get_host())
@@ -74,6 +170,16 @@ def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kw
             return f"{scheme}://{_with_port('archivebox.localhost', req_port)}"
         if _is_local_bind_host(req_host):
             return f"{scheme}://{_with_port('archivebox.localhost', req_port)}"
+        # C) Per-request fallback: when ``BASE_URL`` is unset and CSRF didn't
+        # give us a single origin, trust the request's Host header — but first
+        # peel off any ``admin.`` / ``web.`` / ``api.`` / ``snap-*.`` label.
+        # Otherwise the URL builders below prepend their own role label onto a
+        # host that already carries one, producing the ``snap-X.snap-X.snap-X``
+        # compounding bug. Django has already admitted the host via
+        # ALLOWED_HOSTS; the misconfig banner surfaces the case where the
+        # resulting URL doesn't match what the operator probably intended.
+        canonical_host = strip_role_subdomain(request.get_host())
+        return f"{scheme}://{canonical_host}"
 
     root_host = _root_host_from_listen(config=config)
     return f"{scheme}://{root_host}" if root_host else ""
@@ -113,13 +219,6 @@ def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) ->
     return _build_base_host("api", config=config)
 
 
-def get_public_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_host(config=config)
-    return _build_base_host("public", config=config)
-
-
 def get_snapshot_subdomain(snapshot_id: str) -> str:
     normalized = re.sub(r"[^0-9a-fA-F]", "", snapshot_id or "")
     suffix = (normalized[-12:] if len(normalized) >= 12 else normalized).lower()
@@ -150,6 +249,8 @@ def get_snapshot_lookup_key(snapshot_ref: str) -> str:
     match = _SNAPSHOT_SUBDOMAIN_RE.match(value)
     if match:
         return match.group("suffix")
+    if _SNAPSHOT_ID_RE.match(value):
+        return re.sub(r"[^0-9a-fA-F]", "", value).lower()
     return value
 
 
@@ -221,17 +322,10 @@ def get_api_base_url(request=None, config: dict[str, Any] | None = None, **confi
     return _build_base_url_for_host(_build_base_host("api", request=request, config=config), request=request, config=config)
 
 
-def get_public_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_url(request=request, config=config)
-    return _build_base_url_for_host(_build_base_host("public", request=request, config=config), request=request, config=config)
-
-
 def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_url(get_web_base_url(request=request, config=config), f"/snapshot/{snapshot_id}")
+        return _build_url(get_web_base_url(request=request, config=config), f"/snapshot/{str(snapshot_id).replace('-', '')}")
     return _build_base_url_for_host(
         _build_base_host(get_snapshot_subdomain(snapshot_id), request=request, config=config),
         request=request,

archivebox/core/settings.py

@@ -8,13 +8,12 @@ import importlib
 from pathlib import Path
 
 from django.conf.locale.en import formats as en_formats  # type: ignore
-from django.utils.crypto import get_random_string
 
 import archivebox
 
 from archivebox.config.constants import CONSTANTS
 from archivebox.config.common import get_config
-from archivebox.core.host_utils import normalize_base_url, get_admin_base_url, get_api_base_url
+from archivebox.core.routes_util import get_api_base_url, get_admin_base_url, get_base_url, normalize_base_url
 from .settings_logging import SETTINGS_LOGGING
 
 
@@ -41,7 +40,7 @@ ASGI_APPLICATION = "archivebox.core.asgi.application"
 ROOT_URLCONF = "archivebox.core.urls"
 
 LOGIN_URL = "/accounts/login/"
-LOGOUT_REDIRECT_URL = os.environ.get("LOGOUT_REDIRECT_URL", "/")
+LOGOUT_REDIRECT_URL = CONFIG.LOGOUT_REDIRECT_URL
 
 PASSWORD_RESET_URL = "/accounts/password_reset/"
 APPEND_SLASH = True
@@ -64,23 +63,28 @@ INSTALLED_APPS = [
     # Our ArchiveBox-provided apps (use fully qualified names)
     # NOTE: Order matters! Apps with migrations that depend on other apps must come AFTER their dependencies
     # "archivebox.config",  # ArchiveBox config settings (no models, not a real Django app)
+    "archivebox.plugins",  # plugin discovery, hook helpers, config UI, and plugin metadata views
+    "archivebox.search",  # search backend query helpers, admin search UI, and daemon integrations
     "archivebox.machine",  # handles collecting and storing information about the host machine, network interfaces, binaries, etc.
     "archivebox.workers",  # handles starting and managing background workers and processes (orchestrators and actors)
     "archivebox.personas",  # handles Persona and session management
     "archivebox.core",  # core django model with Snapshot, ArchiveResult, etc. (crawls depends on this)
     "archivebox.crawls",  # handles Crawl and CrawlSchedule models and management (depends on core)
+    "archivebox.progressmonitor",  # live progress endpoint and admin monitor template
     "archivebox.api",  # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
-    # ArchiveBox plugins (hook-based plugins no longer add Django apps)
-    # Use hooks.py discover_hooks() for plugin functionality
+    "abx_plugins.plugins.opencode",
     # 3rd-party apps from PyPI that need to be loaded last
     "admin_data_views",  # handles rendering some convenient automatic read-only views of data in Django admin
     "django_extensions",  # provides Django Debug Toolbar (and other non-debug helpers)
 ]
 
+DJANGO_OBJECT_ACTIONS_DEFAULT_HTTP_METHOD = "POST"
+
 
 MIDDLEWARE = [
     "archivebox.core.middleware.TimezoneMiddleware",
     "django.middleware.security.SecurityMiddleware",
+    "archivebox.core.middleware.AdminCookieIsolationMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.common.CommonMiddleware",
     "archivebox.api.middleware.ApiCorsMiddleware",
@@ -174,9 +178,9 @@ except ImportError:
 
 STATIC_URL = "/static/"
 TEMPLATES_DIR_NAME = "templates"
-CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONFIG.CUSTOM_TEMPLATES_DIR) and os.access(CONFIG.CUSTOM_TEMPLATES_DIR, os.R_OK)
+CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK)
 STATICFILES_DIRS = [
-    *([str(CONFIG.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
+    *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
     # *[
     #     str(plugin_dir / 'static')
     #     for plugin_dir in PLUGIN_DIRS.values()
@@ -187,7 +191,7 @@ STATICFILES_DIRS = [
 ]
 
 TEMPLATE_DIRS = [
-    *([str(CONFIG.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
+    *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
     # *[
     #     str(plugin_dir / 'templates')
     #     for plugin_dir in PLUGIN_DIRS.values()
@@ -210,6 +214,7 @@ TEMPLATES = [
                 "django.template.context_processors.request",
                 "django.contrib.auth.context_processors.auth",
                 "django.contrib.messages.context_processors.messages",
+                "archivebox.core.context_processors.archivebox_globals",
             ],
         },
     },
@@ -224,9 +229,9 @@ TEMPLATES = [
 # CACHE_DB_PATH = CONSTANTS.CACHE_DIR / CACHE_DB_FILENAME
 # CACHE_DB_TABLE = 'django_cache'
 
-DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", str(CONSTANTS.DATABASE_FILE))
-SQLITE_JOURNAL_MODE = os.environ.get("ARCHIVEBOX_SQLITE_JOURNAL_MODE", "WAL")
-SQLITE_MMAP_SIZE = os.environ.get("ARCHIVEBOX_SQLITE_MMAP_SIZE", "0" if CONSTANTS.IN_DOCKER else "134217728")
+DATABASE_NAME = CONFIG.DATABASE_NAME
+SQLITE_JOURNAL_MODE = CONFIG.SQLITE_JOURNAL_MODE
+SQLITE_MMAP_SIZE = CONFIG.SQLITE_MMAP_SIZE
 
 SQLITE_CONNECTION_OPTIONS = {
     "ENGINE": "archivebox.core.sqlite_backend",
@@ -235,7 +240,7 @@ SQLITE_CONNECTION_OPTIONS = {
         # https://gcollazo.com/optimal-sqlite-settings-for-django/
         # https://litestream.io/tips/#busy-timeout
         # https://docs.djangoproject.com/en/5.1/ref/databases/#setting-pragma-options
-        "timeout": 30,
+        "timeout": CONFIG.SQLITE_BUSY_TIMEOUT / 1000,
         "check_same_thread": False,
         # Keep SQLite on Django's default deferred transaction mode. BEGIN
         # IMMEDIATE grabs the write lock as soon as atomic() opens, which is
@@ -246,7 +251,7 @@ SQLITE_CONNECTION_OPTIONS = {
         "transaction_mode": None,
         "init_command": (
             "PRAGMA foreign_keys=ON;"
-            "PRAGMA busy_timeout = 30000;"
+            f"PRAGMA busy_timeout = {CONFIG.SQLITE_BUSY_TIMEOUT};"
             f"PRAGMA journal_mode = {SQLITE_JOURNAL_MODE};"
             "PRAGMA synchronous = NORMAL;"
             "PRAGMA temp_store = MEMORY;"
@@ -275,7 +280,7 @@ MIGRATION_MODULES = {"signal_webhooks": None}
 
 # Django requires DEFAULT_AUTO_FIELD to subclass AutoField (BigAutoField, SmallAutoField, etc.)
 # Cannot use UUIDField here until Django 6.0 introduces DEFAULT_PK_FIELD setting
-# For now: manually add `id = models.UUIDField(primary_key=True, default=uuid7, ...)` to all models
+# For now: manually add `id = CompactUUIDField(primary_key=True, default=uuid7, ...)` to all models
 # OR inherit from ModelWithUUID base class which provides UUID primary key
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 
@@ -333,7 +338,7 @@ STORAGES = {
         "BACKEND": "django.core.files.storage.FileSystemStorage",
         "OPTIONS": {
             "base_url": "/archive/",
-            "location": CONFIG.ARCHIVE_DIR,
+            "location": CONSTANTS.ARCHIVE_DIR,
         },
     },
     # "snapshots": {
@@ -358,7 +363,34 @@ CHANNEL_LAYERS = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"}
 ### Security Settings
 ################################################################################
 
-SECRET_KEY = CONFIG.SECRET_KEY or get_random_string(50, "abcdefghijklmnopqrstuvwxyz0123456789_")
+# Persist SECRET_KEY on first use. Data dirs created before init wrote a
+# SECRET_KEY line — or those whose ArchiveBox.conf was hand-edited to remove
+# it — would otherwise sign session cookies with a fresh random key on every
+# boot (because the pydantic field's default_factory regenerates), logging
+# users out on every server restart. ``archivebox init`` writes this for new
+# collections; this branch is the recovery path for the rest.
+#
+# We can't check ``CONFIG.SECRET_KEY`` for "missing" — pydantic's
+# default_factory already filled it with a fresh random value. We have to
+# inspect the conf file directly to know whether the value will survive.
+SECRET_KEY = CONFIG.SECRET_KEY
+try:
+    from archivebox.config.configset import BaseConfigSet as _BaseConfigSet
+
+    _persisted_keys = _BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
+    _secret_persisted = bool((_persisted_keys.get("SECRET_KEY") or "").strip())
+except Exception:
+    _secret_persisted = True  # err on the side of NOT touching disk
+if not _secret_persisted:
+    try:
+        from archivebox.config.collection import write_config_file
+
+        write_config_file({"SECRET_KEY": SECRET_KEY})
+    except Exception:
+        # Read-only mount, missing data dir, mid-init race — fall back to the
+        # in-memory random key. The user will get logged out on the next boot
+        # but the server still comes up.
+        pass
 
 ALLOWED_HOSTS = [host.strip() for host in CONFIG.ALLOWED_HOSTS.split(",") if host.strip()]
 CSRF_TRUSTED_ORIGINS = list({origin.strip() for origin in CONFIG.CSRF_TRUSTED_ORIGINS.split(",") if origin.strip()})
@@ -371,21 +403,49 @@ api_base_url = normalize_base_url(get_api_base_url())
 if api_base_url and api_base_url not in CSRF_TRUSTED_ORIGINS:
     CSRF_TRUSTED_ORIGINS.append(api_base_url)
 
-# automatically fix case when user sets ALLOWED_HOSTS (e.g. to archivebox.example.com)
-# but forgets to add https://archivebox.example.com to CSRF_TRUSTED_ORIGINS
+# Auto-extend CSRF_TRUSTED_ORIGINS with the explicit ALLOWED_HOSTS entries so
+# users who set ALLOWED_HOSTS=archivebox.example.com don't have to also list
+# https://archivebox.example.com under CSRF_TRUSTED_ORIGINS. (The previous
+# per-host WARNING print was just noise — the auto-append below is the actual
+# fix, and the effective CSRF_TRUSTED_ORIGINS gets surfaced once at startup
+# from archivebox_server.py.)
 for hostname in ALLOWED_HOSTS:
+    if hostname == "*":
+        continue
     https_endpoint = f"https://{hostname}"
-    if hostname != "*" and https_endpoint not in CSRF_TRUSTED_ORIGINS:
-        print(f"[!] WARNING: {https_endpoint} from ALLOWED_HOSTS should be added to CSRF_TRUSTED_ORIGINS")
+    if https_endpoint not in CSRF_TRUSTED_ORIGINS:
         CSRF_TRUSTED_ORIGINS.append(https_endpoint)
 
 SECURE_BROWSER_XSS_FILTER = True
 SECURE_CONTENT_TYPE_NOSNIFF = True
 SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
 
-CSRF_COOKIE_SECURE = False
-SESSION_COOKIE_SECURE = False
+# When BASE_URL is an https:// URL the deployment is HTTPS end-to-end, typically
+# behind a TLS-terminating proxy/tunnel (the bundled traefik/cloudflared profiles,
+# or your own caddy/traefik/nginx) where the proxy -> archivebox hop is plain HTTP, so
+# request.is_secure() / request.scheme would otherwise report http. Honour the
+# proxy's X-Forwarded-Proto so request-derived schemes are correct, and mark the
+# admin session + CSRF cookies Secure so auth cookies are never sent in cleartext.
+# Derived from the RESOLVED base URL's scheme — no separate flag to keep in sync.
+# get_base_url() also covers deployments that only set CSRF_TRUSTED_ORIGINS (the
+# implicit-BASE_URL fallback used on 0.7.x->0.9.x upgrades), so HTTPS hardening
+# isn't lost until BASE_URL is migrated. A plain-http base (e.g. local
+# http://archivebox.localhost:8000) keeps the defaults below.
+BASE_URL_IS_HTTPS = get_base_url(config=CONFIG).strip().lower().startswith("https://")
+if BASE_URL_IS_HTTPS:
+    SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+CSRF_COOKIE_SECURE = BASE_URL_IS_HTTPS
+SESSION_COOKIE_SECURE = BASE_URL_IS_HTTPS
 SESSION_COOKIE_HTTPONLY = True
+SESSION_COOKIE_NAME = f"archivebox_sessionid_{CONSTANTS.COLLECTION_ID}"
+CSRF_COOKIE_NAME = f"archivebox_csrftoken_{CONSTANTS.COLLECTION_ID}"
+# Auth cookies are intentionally scoped to the exact host that set them so
+# the admin session is NOT readable from web.* / api.* — that
+# split is a security boundary, not a UX choice. Subdomains that need to
+# render an "is the user logged in?" indicator must use the single-bit
+# `archivebox_admin_logged_in` hint cookie set by core/middleware.py
+# (which IS scoped to the listen-host parent), never widen these.
 SESSION_COOKIE_DOMAIN = None
 CSRF_COOKIE_DOMAIN = None
 SESSION_COOKIE_AGE = 1209600  # 2 weeks
@@ -492,11 +552,11 @@ ADMIN_DATA_VIEWS = {
         },
         {
             "route": "plugins/",
-            "view": "archivebox.config.views.plugins_list_view",
+            "view": "archivebox.plugins.views.plugins_list_view",
             "name": "Plugins",
             "items": {
                 "route": "<str:key>/",
-                "view": "archivebox.config.views.plugin_detail_view",
+                "view": "archivebox.plugins.views.plugin_detail_view",
                 "name": "plugin",
             },
         },

archivebox/core/settings_logging.py

@@ -7,6 +7,7 @@ import logging
 
 
 from archivebox.config import CONSTANTS
+from archivebox.misc.logging import STDERR
 
 
 IGNORABLE_URL_PATTERNS = [
@@ -42,6 +43,65 @@ class NoisyRequestsFilter(logging.Filter):
         return True
 
 
+class DaphneCloseTimeoutFilter(logging.Filter):
+    """Drop daphne's noisy "killed slow response after client disconnect" warning.
+
+    Daphne emits this whenever a request handler is still running when the
+    client disconnects (e.g. iframe gets navigated away mid-response while
+    fetching favicon / screenshot / preview html). For our use case these are
+    always benign — the disconnect is the browser cancelling a request, not a
+    server-side fault — so we suppress them outright rather than spamming
+    WARNING. Other daphne.server lines pass through unchanged.
+    """
+
+    def filter(self, record) -> bool:
+        if record.name != "daphne.server":
+            return True
+        logline = record.getMessage()
+        if (
+            "Application instance" in logline
+            and "for connection <WebRequest" in logline
+            and "took too long to shut down" in logline
+            and "was killed" in logline
+        ):
+            return False
+        return True
+
+
+class AsyncioCancelledShieldFilter(logging.Filter):
+    """Drop asyncio's "CancelledError exception in shielded future" noise.
+
+    When a browser disconnects mid-request, daphne cancels the asgi task and
+    asgiref's ``sync_to_async`` shields the synchronous Django view via
+    ``asyncio.shield(exec_coro)``. The shield wakes up to find its parent
+    cancelled and re-raises ``CancelledError``; asyncio's default exception
+    handler then logs the full traceback via the ``asyncio`` logger. There's
+    nothing the server can do (the client is already gone), so these tracebacks
+    are pure noise and cause hundreds of lines of spam per disconnect.
+
+    We match conservatively: only drop the specific shielded-future message
+    that points back into asgiref's shield path. Other asyncio errors fall
+    through unchanged.
+    """
+
+    def filter(self, record) -> bool:
+        if record.name != "asyncio":
+            return True
+        logline = record.getMessage()
+        if "CancelledError exception in shielded future" in logline:
+            return False
+        exc_info = record.exc_info
+        if exc_info and exc_info[0] is not None:
+            try:
+                import asyncio as _asyncio
+
+                if issubclass(exc_info[0], _asyncio.CancelledError):
+                    return False
+            except Exception:
+                pass
+        return True
+
+
 class CustomOutboundWebhookLogFormatter(logging.Formatter):
     def format(self, record):
         result = super().format(record)
@@ -100,6 +160,12 @@ SETTINGS_LOGGING = {
         "noisyrequestsfilter": {
             "()": NoisyRequestsFilter,
         },
+        "daphneclosetimeout": {
+            "()": DaphneCloseTimeoutFilter,
+        },
+        "asynciocancelledshield": {
+            "()": AsyncioCancelledShieldFilter,
+        },
         "stripansi": {
             "()": StripANSIColorCodesFilter,
         },
@@ -116,8 +182,10 @@ SETTINGS_LOGGING = {
             "formatter": "rich",
             "level": "DEBUG",
             "markup": False,
+            "enable_link_path": False,
             "rich_tracebacks": False,  # Use standard Python tracebacks (no frame/box)
-            "filters": ["noisyrequestsfilter", "stripansi"],
+            "console": STDERR,
+            "filters": ["noisyrequestsfilter", "daphneclosetimeout", "asynciocancelledshield", "stripansi"],
         },
         "logfile": {
             "level": "INFO",
@@ -126,11 +194,12 @@ SETTINGS_LOGGING = {
             "maxBytes": 1024 * 1024 * 25,  # 25 MB
             "backupCount": 10,
             "formatter": "rich",
-            "filters": ["noisyrequestsfilter", "stripansi"],
+            "filters": ["noisyrequestsfilter", "daphneclosetimeout", "asynciocancelledshield", "stripansi"],
         },
         "outbound_webhooks": {
             "class": "rich.logging.RichHandler",
             "markup": False,
+            "enable_link_path": False,
             "rich_tracebacks": False,  # Use standard Python tracebacks (no frame/box)
             "formatter": "outbound_webhooks",
         },

archivebox/core/shutdown_util.py

@@ -134,7 +134,10 @@ def foreground_shutdown_signals(
         # signal for hard foreground-command shutdown. Server/update/run and
         # other non-interactive commands raise immediately so their finally
         # blocks can stop owned children without prompting.
-        if raise_on_first_signal or already_requested:
+        if already_requested:
+            os.write(sys.stdout.fileno(), f"\n[🛑] Got {sig.name} again, exiting immediately.\n".encode())
+            os._exit(130)
+        if raise_on_first_signal:
             raise KeyboardInterrupt
 
     try:
@@ -143,6 +146,8 @@ def foreground_shutdown_signals(
             signal.signal(sig, raise_keyboard_interrupt)
         yield state
     finally:
+        if state.signal_name and previous_active_state is not None and not previous_active_state.signal_name:
+            previous_active_state.signal_name = state.signal_name
         _active_shutdown_state = previous_active_state
         for sig, previous_handler in previous_handlers.items():
             signal.signal(sig, previous_handler)

archivebox/core/snapshot_status.py

@@ -0,0 +1,25 @@
+__package__ = "archivebox.core"
+
+from django.db.models import QuerySet
+
+
+def snapshot_status_values() -> tuple[str, ...]:
+    from archivebox.core.models import Snapshot
+
+    return tuple(Snapshot.StatusChoices.values)
+
+
+def normalize_snapshot_status(status: str | None) -> str | None:
+    value = str(status or "").strip().lower()
+    if not value:
+        return None
+
+    valid_statuses = snapshot_status_values()
+    if value not in valid_statuses:
+        raise ValueError(f"Invalid snapshot status: {status}. Expected one of: {', '.join(valid_statuses)}")
+    return value
+
+
+def filter_snapshots_by_status(queryset: QuerySet, status: str | None) -> QuerySet:
+    value = normalize_snapshot_status(status)
+    return queryset.filter(status=value) if value else queryset

archivebox/core/sqlite_backend/base.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import sqlite3
 import time
-import os
 from collections.abc import Mapping
 from itertools import tee
 import re
@@ -11,21 +10,16 @@ from django.db.backends.sqlite3.base import DatabaseWrapper as DjangoSQLiteDatab
 from django.db.backends.sqlite3.base import SQLiteCursorWrapper as DjangoSQLiteCursorWrapper
 
 
-def _is_locked_error(error: BaseException) -> bool:
-    from django.db import OperationalError
+def _sqlite_lock_retry_timeout() -> float:
+    from django.conf import settings
 
-    return isinstance(error, (sqlite3.OperationalError, OperationalError)) and "database is locked" in str(error).lower()
+    return settings.CONFIG.SQLITE_LOCK_RETRY_TIMEOUT
 
 
-def _float_env(name: str, default: float) -> float:
-    try:
-        return float(os.environ.get(name, default))
-    except (TypeError, ValueError):
-        return default
+def _sqlite_lock_retry_interval() -> float:
+    from django.conf import settings
 
-
-SQLITE_LOCK_RETRY_TIMEOUT = _float_env("ARCHIVEBOX_SQLITE_LOCK_RETRY_TIMEOUT", 60.0)
-SQLITE_LOCK_RETRY_INTERVAL = _float_env("ARCHIVEBOX_SQLITE_LOCK_RETRY_INTERVAL", 5.0)
+    return settings.CONFIG.SQLITE_LOCK_RETRY_INTERVAL
 
 
 def _format_sql(query: str, params=None) -> str:
@@ -48,27 +42,17 @@ def _format_sql(query: str, params=None) -> str:
     return compact[:260]
 
 
-def _log_locked_database(query: str, params=None, *, attempt: int, elapsed: float) -> None:
+def _log_locked_database(query: str, params=None, *, attempt: int, elapsed: float, retry_interval: float) -> None:
     from rich.console import Console
 
-    from archivebox.misc.db import sqlite_lock_holders
+    from archivebox.misc.db import log_sqlite_lock_holders
 
     console = Console(stderr=True)
-    console.print(f"[yellow][*] SQLite database is locked for {elapsed:.0f}s; retrying in 5s... attempt={attempt}[/yellow]")
+    console.print(
+        f"[yellow][*] SQLite database is locked for {elapsed:.0f}s; retrying in {retry_interval:g}s... attempt={attempt}[/yellow]",
+    )
     console.print(f"[yellow]    Query: {_format_sql(query, params)}[/yellow]")
-    holders = sqlite_lock_holders()
-    if holders:
-        console.print("[yellow]    DB holders:[/yellow]")
-        for holder in holders[:8]:
-            console.print(f"[yellow]    - {holder}[/yellow]")
-        if len(holders) > 8:
-            console.print(f"[yellow]    ... {len(holders) - 8} more[/yellow]")
-    else:
-        console.print("[yellow]    No local process with index.sqlite3 open was visible to this user.[/yellow]")
-    if attempt == 1:
-        console.print(
-            "[dim]    SQLite does not expose the active SQL statement from another process; only local PIDs with the DB open can be shown.[/dim]",
-        )
+    log_sqlite_lock_holders(console)
 
 
 def _connection_in_transaction(connection) -> bool:
@@ -109,20 +93,24 @@ def _retry_locked_database(action, query: str, params=None, *, db_wrapper=None):
         try:
             return action()
         except (sqlite3.OperationalError, Exception) as err:
-            if not _is_locked_error(err):
+            from archivebox.misc.db import sqlite_lock_error
+
+            if not sqlite_lock_error(err):
                 raise
             attempt += 1
             elapsed = time.monotonic() - started_at
-            _log_locked_database(query, params, attempt=attempt, elapsed=elapsed)
+            retry_timeout = _sqlite_lock_retry_timeout()
+            retry_interval = _sqlite_lock_retry_interval()
+            _log_locked_database(query, params, attempt=attempt, elapsed=elapsed, retry_interval=retry_interval)
             # If SQLite raised while Django is in autocommit mode, do not keep a
             # partially-open sqlite transaction around while waiting. Explicit
             # transaction.atomic() callers keep their normal transaction boundary.
             _recover_non_atomic_connection(db_wrapper, query)
             if _is_inside_atomic(db_wrapper):
                 raise
-            if SQLITE_LOCK_RETRY_TIMEOUT and elapsed >= SQLITE_LOCK_RETRY_TIMEOUT:
+            if retry_timeout and elapsed >= retry_timeout:
                 _abort_locked_database(query, params, elapsed=elapsed, db_wrapper=db_wrapper)
-            time.sleep(SQLITE_LOCK_RETRY_INTERVAL)
+            time.sleep(retry_interval)
 
 
 class SQLiteCursorWrapper(DjangoSQLiteCursorWrapper):