release: archivebox 0.9.35rc15

Updated copyright year in the LICENSE file.

.claude/settings.json

@@ -1,5 +0,0 @@
-{
-  "enabledPlugins": {
-    "pyright-lsp@claude-plugins-official": true
-  }
-}

.claude/settings.local.json

@@ -1,51 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Read(**)",
-      "Glob(**)",
-      "Grep(**)",
-      "Bash(python -m archivebox:*)",
-      "Bash(ls:*)",
-      "Bash(xargs:*)",
-      "Bash(python -c:*)",
-      "Bash(printf:*)",
-      "Bash(pkill:*)",
-      "Bash(python3:*)",
-      "Bash(sqlite3:*)",
-      "WebFetch(domain:github.com)",
-      "Bash(uv add:*)",
-      "Bash(mkdir:*)",
-      "Bash(chmod:*)",
-      "Bash(python -m forum_dl:*)",
-      "Bash(archivebox manage migrate:*)",
-      "Bash(cat:*)",
-      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
-      "Bash(forum-dl:*)",
-      "Bash(pip uninstall:*)",
-      "Bash(python:*)",
-      "Bash(source .venv/bin/activate)",
-      "Bash(mv:*)",
-      "Bash(echo:*)",
-      "Bash(grep:*)",
-      "WebFetch(domain:python-statemachine.readthedocs.io)",
-      "Bash(./bin/run_plugin_tests.sh:*)",
-      "Bash(done)",
-      "Bash(coverage erase:*)",
-      "Bash(gh api:*)"
-    ]
-  },
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Bash",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
-            "statusMessage": "Checking working directory..."
-          }
-        ]
-      }
-    ]
-  }
-}

.dockerignore

@@ -3,18 +3,29 @@
 /.heartbeat.json
 /package.json
 /package-lock.json
+__pycache__
+**/__pycache__
 *.pyc
-__pycache__/
+*.pyo
+*.py[cod]
+*$py.class
 .mypy_cache/
 .pytest_cache/
+.ruff_cache/
+.uv-cache/
 .github/
 .pdm-build/
 .pdm-python
 .eggs/
 .git/
+!.git/
+.git/*
 .vscode/
 !.git/HEAD
-!.git/refs/heads/*
+!.git/packed-refs
+!.git/refs/
+!.git/refs/heads/
+!.git/refs/heads/**
 
 venv/
 .venv/
@@ -38,6 +49,8 @@ snapshots/
 logs/
 archivebox-docker-smoke*/
 archivebox-compose-smoke*/
+docker-test/
+docker-test*/
 core
 *.core
 

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -69,7 +69,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:

.github/ISSUE_TEMPLATE/2-feature_request.yml

@@ -71,7 +71,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:

.github/workflows/debian.yml

@@ -1,214 +0,0 @@
-name: Build Debian package
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'pkg/debian/**'
-      - 'bin/build_deb.sh'
-      - 'bin/release_deb.sh'
-      - '.github/workflows/debian.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        arch: [amd64, arm64]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Install nfpm
-        env:
-          NFPM_VERSION: "2.45.1"
-        run: |
-          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \
-            | sudo tar -xz -C /usr/local/bin nfpm
-          nfpm --version
-
-      - name: Build .deb package
-        run: |
-          export VERSION="${{ steps.version.outputs.version }}"
-          export ARCH="${{ matrix.arch }}"
-          ./bin/build_deb.sh
-
-      - name: Verify .deb package contents
-        run: |
-          DEB_FILE="$(ls dist/archivebox*.deb | head -1)"
-          echo "=== Package info ==="
-          dpkg-deb --info "$DEB_FILE"
-          echo ""
-          echo "=== Package contents ==="
-          dpkg-deb --contents "$DEB_FILE"
-          echo ""
-          echo "=== Control fields ==="
-          dpkg-deb --field "$DEB_FILE"
-
-      - name: Upload .deb artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb
-          path: dist/*.deb
-
-  test:
-    needs: build
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-            runner: ubuntu-24.04
-          - arch: arm64
-            runner: ubuntu-24.04-arm
-    runs-on: ${{ matrix.runner }}
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local wheel
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --wheel --out-dir /tmp/wheels/
-
-      - name: Download .deb artifact
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*-${{ matrix.arch }}.deb
-          merge-multiple: true
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget
-
-      - name: Pre-seed virtualenv with local wheel before dpkg install
-        run: |
-          # CI-only: pre-seed the venv with a local wheel so we test the
-          # unreleased code, not whatever is on PyPI. We explicitly use python3.13
-          # here (matching the system dep installed above) to ensure the venv is
-          # created with the correct Python version. On real installs, install.sh
-          # handles this by preferring python3.13 and failing if < 3.13.
-          # When postinstall.sh runs during dpkg -i, it finds the venv already
-          # populated and just upgrades deps.
-          sudo mkdir -p /opt/archivebox
-          sudo python3.13 -m venv /opt/archivebox/venv
-          sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools
-          sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl
-          echo "[√] Pre-seeded /opt/archivebox/venv with local wheel"
-
-      - name: Install .deb package
-        run: |
-          # dpkg install will run postinstall which creates the user,
-          # sets up systemd, and tries pip install (which finds it already installed)
-          sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y
-
-      - name: Verify archivebox is installed
-        run: |
-          which archivebox
-          archivebox version
-
-      - name: Verify wrapper script works
-        run: |
-          /usr/bin/archivebox version
-          /usr/bin/archivebox --help | head -5
-
-      - name: Test archivebox init as archivebox user
-        run: |
-          # The postinstall should have created the user
-          id archivebox
-          sudo mkdir -p /tmp/archivebox-test
-          sudo chown archivebox:archivebox /tmp/archivebox-test
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init'
-
-      - name: Test archivebox status
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status'
-
-      - name: Test archivebox add
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"'
-
-      - name: Verify systemd service file exists
-        run: |
-          test -f /usr/lib/systemd/system/archivebox.service
-          cat /usr/lib/systemd/system/archivebox.service
-
-  # Upload .deb to GitHub Release only after tests pass
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: [build, test]
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: write
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Determine release tag
-        id: tag
-        run: |
-          TAG="${{ github.event.release.tag_name }}"
-          if [ -z "$TAG" ]; then
-            TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')"
-            echo "[i] No release tag in event context, using version from pyproject.toml: $TAG"
-          fi
-          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
-
-      - name: Download all .deb artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*.deb
-          merge-multiple: true
-
-      - name: Upload .deb to GitHub Release
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          TAG="${{ steps.tag.outputs.tag }}"
-          # Verify the release exists before uploading
-          if ! gh release view "$TAG" >/dev/null 2>&1; then
-            echo "[!] No GitHub Release found for tag $TAG."
-            if [ -n "${{ github.event.release.tag_name }}" ]; then
-              echo "[X] This was triggered by a release event — the release should exist. Failing."
-              exit 1
-            fi
-            echo "[i] Skipping upload (workflow_dispatch without a release)."
-            echo "    Create a release first or trigger via the release event."
-            exit 0
-          fi
-          gh release upload "$TAG" *.deb --clobber

.github/workflows/docker.yml

@@ -5,7 +5,7 @@ on:
   workflow_call:
   push:
     branches:
-      - dev
+      - "**"
     tags:
       - 'v*'
   # pull_request:
@@ -13,13 +13,14 @@ on:
 env:
   DOCKERHUB_IMAGE: archivebox/archivebox
   GHCR_IMAGE: ghcr.io/archivebox/archivebox
+  ABX_DL_IMAGE: archivebox/abx-dl:latest
 
 permissions:
   contents: read
   packages: write
 
 concurrency:
-  group: docker-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -45,6 +46,44 @@ jobs:
         #   submodules: true
         #   fetch-depth: 1
 
+      - name: Wait for released ArchiveBox deps on PyPI
+        run: |
+          python3 - <<'PY'
+          import json
+          import re
+          import sys
+          import time
+          import tomllib
+          import urllib.request
+
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp:
+                      releases = json.load(resp)["releases"]
+                  if version in releases:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
+          PY
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
@@ -58,6 +97,24 @@ jobs:
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
+      - name: Wait for published abx-dl image
+        id: abx_dl_image
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          deadline=$((SECONDS + 1800))
+          until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
+            if (( SECONDS >= deadline )); then
+              echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2
+              exit 1
+            fi
+            echo "${ABX_DL_IMAGE} is not published yet; waiting..."
+            sleep 30
+          done
+
+          echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
+          docker buildx imagetools inspect "${ABX_DL_IMAGE}"
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         if: github.event_name != 'pull_request'
@@ -78,13 +135,10 @@ jobs:
         run: |
           set -Eeuo pipefail
           VERSION="$(python3 - <<'PY'
-          from pathlib import Path
-          import re
+          import tomllib
 
-          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
-          if not match:
-              raise SystemExit("Failed to read version from pyproject.toml")
-          print(match.group(1))
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
           PY
           )"
 
@@ -93,11 +147,12 @@ jobs:
             echo "org.opencontainers.image.version=${VERSION}"
             echo "org.opencontainers.image.revision=${GITHUB_SHA}"
             echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
+            echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
             echo 'EOF'
             echo "version=${VERSION}"
           } >> "$GITHUB_OUTPUT"
 
-          echo "[+] Building ${{ matrix.platform }} for ${VERSION}"
+          echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
 
       - name: Build and push digest
         id: docker_build
@@ -111,6 +166,8 @@ jobs:
             ${{ env.DOCKERHUB_IMAGE }}
             ${{ env.GHCR_IMAGE }}
           labels: ${{ steps.docker_meta.outputs.labels }}
+          build-args: |
+            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
           cache-from: type=gha,scope=${{ matrix.cache_scope }}
           cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
           platforms: ${{ matrix.platform }}
@@ -177,36 +234,49 @@ jobs:
         run: |
           set -Eeuo pipefail
           VERSION="$(python3 - <<'PY'
-          from pathlib import Path
-          import re
+          import tomllib
 
-          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
-          if not match:
-              raise SystemExit("Failed to read version from pyproject.toml")
-          print(match.group(1))
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
           PY
           )"
-          SHORT_SHA="${GITHUB_SHA::8}"
+          BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)"
+          SHORT_SHA="${GITHUB_SHA::12}"
+          test -n "$BRANCH_TAG"
+          test -n "$SHORT_SHA"
 
           {
             echo 'dockerhub_tags<<EOF'
-            echo "${DOCKERHUB_IMAGE}:dev"
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${DOCKERHUB_IMAGE}:latest"
+            fi
+            echo "${DOCKERHUB_IMAGE}:${BRANCH_TAG}"
             echo "${DOCKERHUB_IMAGE}:${VERSION}"
             echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
             echo 'EOF'
             echo 'ghcr_tags<<EOF'
-            echo "${GHCR_IMAGE}:dev"
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${GHCR_IMAGE}:latest"
+            fi
+            echo "${GHCR_IMAGE}:${BRANCH_TAG}"
             echo "${GHCR_IMAGE}:${VERSION}"
             echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
             echo 'EOF'
             echo "version=${VERSION}"
+            echo "branch_tag=${BRANCH_TAG}"
             echo "short_sha=${SHORT_SHA}"
           } >> "$GITHUB_OUTPUT"
 
           echo "[+] Publishing Docker Hub tags:"
-          printf '%s\n' "${DOCKERHUB_IMAGE}:dev" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${DOCKERHUB_IMAGE}:latest"
+          fi
+          printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
           echo "[+] Publishing GHCR tags:"
-          printf '%s\n' "${GHCR_IMAGE}:dev" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${GHCR_IMAGE}:latest"
+          fi
+          printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
 
       - name: Create Docker Hub manifest
         shell: bash
@@ -250,8 +320,12 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
-          docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
-          docker buildx imagetools inspect "${GHCR_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
+          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
+          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
 
       - name: Update README
         uses: peter-evans/dockerhub-description@v4

.github/workflows/homebrew.yml

@@ -1,233 +0,0 @@
-name: Build Homebrew formula
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'brew_dist/**'
-      - 'bin/build_brew.sh'
-      - 'bin/release_brew.sh'
-      - '.github/workflows/homebrew.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-and-test:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-latest, ubuntu-24.04]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Validate formula template syntax
-        run: ruby -c brew_dist/archivebox.rb
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies (Linux)
-        if: runner.os == 'Linux'
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local sdist
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --sdist --out-dir /tmp/sdist/
-
-      - name: Generate formula from local sdist
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)"
-          SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')"
-
-          # Install archivebox + poet into a temp venv to generate resource stanzas
-          python3 -m venv /tmp/poet-venv
-          source /tmp/poet-venv/bin/activate
-          pip install --quiet "$SDIST_PATH" homebrew-pypi-poet
-          echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
-          RESOURCES="$(poet archivebox)"
-          deactivate
-
-          # For CI: use file:// URL pointing to local sdist
-          # For release: this gets overridden with the PyPI URL
-          SDIST_URL="file://${SDIST_PATH}"
-
-          cat > /tmp/archivebox.rb << RUBY
-class Archivebox < Formula
-  include Language::Python::Virtualenv
-
-  desc "Self-hosted internet archiving solution"
-  homepage "https://github.com/ArchiveBox/ArchiveBox"
-  url "${SDIST_URL}"
-  sha256 "${SDIST_SHA256}"
-  license "MIT"
-
-  depends_on "python@3.13"
-
-${RESOURCES}
-
-  def install
-    virtualenv_install_with_resources
-  end
-
-  def post_install
-    data_dir = var/"archivebox"
-    data_dir.mkpath
-    ENV["DATA_DIR"] = data_dir.to_s
-    system bin/"archivebox", "init"
-  end
-
-  def caveats
-    <<~EOS
-      ArchiveBox data is stored in:
-        #{var}/archivebox
-
-      To start archiving, run:
-        cd #{var}/archivebox && archivebox add 'https://example.com'
-
-      To start the web UI:
-        cd #{var}/archivebox && archivebox server 0.0.0.0:8000
-    EOS
-  end
-
-  test do
-    assert_match version.to_s, shell_output("#{bin}/archivebox version")
-  end
-end
-RUBY
-
-          echo "[√] Generated formula:"
-          ruby -c /tmp/archivebox.rb
-          cat /tmp/archivebox.rb
-
-      - name: Install Homebrew (Linux only)
-        if: runner.os == 'Linux'
-        run: |
-          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
-          echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV"
-          echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV"
-          echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV"
-          echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH"
-          echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH"
-
-      - name: Install brew dependencies
-        run: brew install python@3.13
-
-      - name: Install archivebox via brew from local formula
-        run: |
-          brew install --build-from-source --verbose /tmp/archivebox.rb
-
-      - name: Verify archivebox version
-        run: archivebox version
-
-      - name: Test archivebox init
-        run: |
-          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
-          archivebox init
-
-      - name: Test archivebox status
-        run: |
-          cd /tmp/archivebox-test
-          archivebox status
-
-  # On release only: generate the real formula with PyPI URL and push to tap
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: build-and-test
-    runs-on: macos-latest
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Wait for PyPI package availability
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..."
-          for i in $(seq 1 30); do
-            if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then
-              echo "[√] archivebox==${VERSION} is available on PyPI"
-              break
-            fi
-            if [ "$i" -eq 30 ]; then
-              echo "[!] Timed out waiting for PyPI. Trying to install anyway..."
-              break
-            fi
-            echo "    Attempt $i/30 - not yet available, waiting 30s..."
-            sleep 30
-          done
-
-      - name: Generate release formula via build_brew.sh
-        run: ./bin/build_brew.sh
-
-      - name: Test formula install
-        run: |
-          brew install --build-from-source brew_dist/archivebox.rb
-          archivebox version
-
-      - name: Upload formula artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox.rb
-          path: brew_dist/archivebox.rb
-
-      - name: Push to homebrew-archivebox tap
-        env:
-          GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap
-
-          cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \
-            cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb
-
-          cd /tmp/tap
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add -A
-          if git diff --cached --quiet; then
-            echo "[i] No changes to formula, skipping push."
-          else
-            git commit -m "Update archivebox to v${VERSION}"
-            git push origin HEAD
-            echo "[√] Formula pushed to homebrew-archivebox tap"
-          fi

.github/workflows/lint.yml

@@ -39,12 +39,13 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - name: Install dependencies with uv
         run: |
-          uv sync --extra sonic --extra debug --all-groups --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --all-groups -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[sonic,debug]"
 
       - name: Run prek
         run: uv run --no-sync --no-sources prek run --all-files

.github/workflows/pip.yml

@@ -22,12 +22,14 @@ jobs:
       id-token: write
 
     runs-on: ubuntu-24.04
+    environment: pypi
     steps:
       - uses: actions/checkout@v4
 
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - name: Set up Python
@@ -42,6 +44,44 @@ jobs:
           packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
           version: 1.0
 
+      - name: Wait for released ArchiveBox deps on PyPI
+        run: |
+          python - <<'PY'
+          import json
+          import re
+          import sys
+          import time
+          import tomllib
+          import urllib.request
+
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp:
+                      releases = json.load(resp)["releases"]
+                  if version in releases:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
+          PY
+
       - name: UV install archivebox dev + run sub-dependencies
         run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
 

.github/workflows/release-runner.yml

@@ -30,6 +30,7 @@ jobs:
 
       - uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - uses: actions/setup-node@v4

.github/workflows/release.yml

@@ -2,8 +2,8 @@ name: Release
 
 # Orchestrates the full release pipeline:
 #   1. Build and publish pip package to PyPI
-#   2. Build .deb packages and Homebrew formula (in parallel, after pip)
-#   3. Build Docker images (in parallel with deb/brew)
+#   2. Build Homebrew formula after pip
+#   3. Build Docker images in parallel
 #
 # Individual workflows also run on push for CI (see their own triggers).
 # Manual fallback for the hosted release pipeline. The normal prerelease loop
@@ -28,12 +28,6 @@ jobs:
     uses: ./.github/workflows/pip.yml
     secrets: inherit
 
-  debian:
-    name: Build .deb packages
-    needs: pip
-    uses: ./.github/workflows/debian.yml
-    secrets: inherit
-
   homebrew:
     name: Update Homebrew formula
     needs: pip

.github/workflows/test-parallel.yml

@@ -49,8 +49,16 @@ jobs:
 
             # Extract a display name for the test
             name="main/$(basename $test_file .py | sed 's/^test_//')"
+            needs_chromium=false
+            if grep -Eiq '(chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue)' "$test_file"; then
+              needs_chromium=true
+            fi
+            needs_sonic=false
+            if grep -Eiq "shutil\\.which\\([\"']sonic|SEARCH_BACKEND_ENGINE=.*sonic|worker_sonic" "$test_file"; then
+              needs_sonic=true
+            fi
 
-            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
           done
           json_array+="]"
 
@@ -79,8 +87,12 @@ jobs:
             if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
               needs_chromium=true
             fi
+            needs_sonic=false
+            if [ "$plugin_name" = "search_backend_sonic" ]; then
+              needs_sonic=true
+            fi
 
-            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium}"
+            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
           done
           json_array+="]"
 
@@ -129,7 +141,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -151,13 +163,79 @@ jobs:
 
       - name: Install dependencies with uv
         run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
+
+      - name: Install Chrome through ArchiveBox
+        if: ${{ matrix.test.needs_chromium }}
+        env:
+          MIN_CHROMIUM_MAJOR: "149"
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
+          case "$major" in
+            ''|*[!0-9]*) major=0 ;;
+          esac
+          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
+            exit 1
+          fi
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
+          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+
+      - name: Install Sonic through ArchiveBox
+        if: ${{ matrix.test.needs_sonic }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
+          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
 
       - name: Run test - ${{ matrix.test.name }}
         run: |
           mkdir -p tests/out
+          set +e
           uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
+          status=$?
+          set -e
+          if [ "$status" -eq 5 ]; then
+            echo "No tests collected from ${{ matrix.test.path }}; treating as an empty test module."
+            exit 0
+          fi
+          exit "$status"
 
   plugin-tests:
     name: ${{ matrix.plugin.name }}
@@ -199,7 +277,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -221,58 +299,28 @@ jobs:
 
       - name: Install dependencies with uv
         run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
 
-      - name: Select Chromium for extension-capable CDP
+      - name: Install Chrome through ArchiveBox
         if: ${{ matrix.plugin.needs_chromium }}
         env:
           MIN_CHROMIUM_MAJOR: "149"
           LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
         run: |
           set -euo pipefail
-          mkdir -p "$LIB_DIR"
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
           echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
           echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
 
-          candidate="/usr/bin/chromium"
-          version=""
-          major=0
-          if [ -x "$candidate" ]; then
-            version="$("$candidate" --version || true)"
-            major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
-            case "$major" in
-              ''|*[!0-9]*) major=0 ;;
-            esac
-          fi
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
 
-          if [ "$major" -ge "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Using runner Chromium: $candidate ($version)"
-            echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
-            exit 0
-          fi
-
-          echo "Runner Chromium is unavailable or too old: ${version:-not found}"
-          echo "Installing Playwright Chromium into $LIB_DIR..."
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=npm \
-            --lib="$LIB_DIR" \
-            --postinstall-scripts=false \
-            --min-release-age=0 \
-            --no-cache \
-            --overrides='{"npm":{"install_args":["playwright@next"]}}' \
-            playwright
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=playwright \
-            --lib="$LIB_DIR" \
-            --bin-dir="$LIB_DIR/env/bin" \
-            --min-version="$MIN_CHROMIUM_MAJOR" \
-            chromium
-
-          candidate="$LIB_DIR/env/bin/chromium"
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
           if [ ! -x "$candidate" ]; then
-            echo "Playwright Chromium shim was not created at $candidate" >&2
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
             exit 1
           fi
           version="$("$candidate" --version || true)"
@@ -281,12 +329,37 @@ jobs:
             ''|*[!0-9]*) major=0 ;;
           esac
           if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Playwright Chromium is still too old: ${version:-unknown}" >&2
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
             exit 1
           fi
-          echo "Using Playwright Chromium: $candidate ($version)"
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
           echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
 
+      - name: Install Sonic through ArchiveBox
+        if: ${{ matrix.plugin.needs_sonic }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
+          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
+
       - name: Run plugin tests - ${{ matrix.plugin.name }}
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

.github/workflows/test.yml

@@ -49,7 +49,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v4
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -72,8 +72,8 @@ jobs:
 
       - name: Install dependencies with uv
         run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
 
       ### Run the tests
       - name: Directory listing for debugging

.github/workflows/update-homebrew-tap.yml

@@ -0,0 +1,35 @@
+name: Update Homebrew tap
+
+on:
+  push:
+    branches: [dev]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: update-homebrew-tap-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dispatch:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Dispatch tap update
+        env:
+          HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
+        run: |
+          if [ -z "$HOMEBREW_TAP_TOKEN" ]; then
+            echo "HOMEBREW_TAP_TOKEN is not configured; tap hourly fallback will update the formula."
+            exit 0
+          fi
+
+          curl --fail-with-body \
+            -X POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${HOMEBREW_TAP_TOKEN}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/ArchiveBox/homebrew-archivebox/dispatches \
+            -d '{"event_type":"archivebox-dev-updated"}'

AGENTS.md

@@ -0,0 +1,14 @@
+I found 31 unique `no mocking` prompts in recent Codex history, across 21 session transcripts. Only one transcript literally started with `no mocking`; most had it later as testing guidance. Consolidated advice:
+
+- Tests must hit real user-facing code paths: CLI commands, REST/API calls, browser UI, real hooks, real ArchiveBox data dirs, real pytest fixtures, and real subprocess/binary behavior.
+- No mocking, faking, simulating, monkey patching, handwritten fake objects, fake buses, fake hooks, fake binaries, fake handlers, or direct-post shortcuts when the user path is through UI/extension/CLI.
+- No skipped, xfailed, flaky, or “works around platform” tests. Flakiness is treated as a bug, especially on macOS/browser flows.
+- Prefer live integration tests over narrow unit tests when behavior depends on browsers, binaries, ArchiveBox crawls, plugins, LLMs, or server state.
+- Assertions must validate real correctness: returned values, exit codes, DB rows, filesystem contents, field values, uploaded files, rendered output, and side effects. “No error occurred” or “attribute exists” is not enough.
+- Start fixes with failing red tests that reproduce the missing behavior or regression, then implement the minimal fix and confirm the test passes.
+- Use realistic setup patterns “like a user would”: events + bus + handlers, real browser pages/CDP sessions, real URLs or `pytest-httpserver`, real rows, real snapshots, real installs, real local browser/server state.
+- For ArchiveBox/API tests, use existing `conftest.py` fixtures and test harnesses, real test DB rows/data dirs, and user-facing commands/APIs rather than bespoke helpers.
+- For browser/extension tests, trigger behavior through the real extension UI or actual browser session, not direct posting or mocked browser/session objects.
+- For binary/provider tests, use real binaries and real installs; verify constraints and final installed package metadata, not just install success.
+- For coverage quality, keep tests strict, deterministic, grouped consistently, and use a few larger realistic tests when that gives better surface coverage than many tiny fake unit tests.
+- Avoid weakening test coverage, adding compatibility/shim/fallback layers, or guessing from code shape. Trace root causes, verify assumptions with tests/scripts, and let real type/parse errors surface normally.

CLAUDE.md

@@ -1,497 +0,0 @@
-# Claude Code Development Guide for ArchiveBox
-
-## Quick Start
-
-```bash
-# Set up dev environment (always use uv, never pip directly)
-uv sync --dev --all-extras
-
-# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
-```
-
-## Development Environment Setup
-
-### Prerequisites
-- Python 3.11+ (3.13 recommended)
-- uv package manager
-- A non-root user for running tests (e.g., `testuser`)
-
-### Install Dependencies
-```bash
-uv sync --dev --all-extras  # Always use uv, never pip directly
-```
-
-### Activate Virtual Environment
-```bash
-source .venv/bin/activate
-```
-
-### Common Gotchas
-
-#### File Permissions
-New files created by root need permissions fixed for testuser:
-```bash
-chmod 644 archivebox/tests/test_*.py
-```
-
-#### DATA_DIR Environment Variable
-ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
-
-## Code Style Guidelines
-
-### Naming Conventions for Grep-ability
-Use consistent naming for everything to enable easy grep-ability and logical grouping:
-
-**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
-
-**Examples**:
-```python
-# Filesystem migration methods - all start with fs_
-def fs_migration_needed() -> bool: ...
-def fs_migrate() -> None: ...
-def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
-def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
-def _fs_next_version(current: str) -> str: ...
-
-# Logging methods - ALL must start with log_ or _log
-def log_migration_start(snapshot_id: str) -> None: ...
-def _log_error(message: str) -> None: ...
-def log_validation_result(ok: bool, msg: str) -> None: ...
-```
-
-**Rules**:
-- Group related functions with common prefixes
-- Use `_` prefix for internal/private helpers within the same family
-- ALL logging-related methods MUST start with `log_` or `_log`
-- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
-- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
-
-### Minimize Unique Names and Data Structures
-**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
-
-**Example - GOOD**:
-```python
-# Binary has overrides field
-binary = Binary(overrides={'TIMEOUT': '60s'})
-
-# Binary reuses the same field name and structure
-class Binary(models.Model):
-    overrides = models.JSONField(default=dict)  # Same name, same structure
-```
-
-**Example - BAD**:
-```python
-# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
-class Binary(models.Model):
-    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
-```
-
-**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
-
-## Testing
-
-### CRITICAL: Never Run as Root
-ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
-
-```bash
-# Run all migration tests
-sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
-
-# Run specific test file
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
-
-# Run single test
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
-```
-
-### Test File Structure
-```
-archivebox/tests/
-├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
-├── test_migrations_fresh.py      # Fresh install tests
-├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
-├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
-└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
-```
-
-### Test Writing Standards
-
-#### NO MOCKS - Real Tests Only
-Tests must exercise real code paths:
-- Create real SQLite databases with version-specific schemas
-- Seed with realistic test data
-- Run actual `python -m archivebox` commands via subprocess
-- Query SQLite directly to verify results
-
-**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
-
-#### NO SKIPS
-Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
-
-#### Strict Assertions
-- `init` command must return exit code 0 (not `[0, 1]`)
-- Verify ALL data is preserved, not just "at least one"
-- Use exact counts (`==`) not loose bounds (`>=`)
-
-### Example Test Pattern
-```python
-def test_migration_preserves_snapshots(self):
-    """Migration should preserve all snapshots."""
-    result = run_archivebox(self.work_dir, ['init'], timeout=45)
-    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
-
-    ok, msg = verify_snapshot_count(self.db_path, expected_count)
-    self.assertTrue(ok, msg)
-```
-
-### Testing Gotchas
-
-#### Extractors Disabled for Speed
-Tests disable all extractors via environment variables for faster execution:
-```python
-env['SAVE_TITLE'] = 'False'
-env['SAVE_FAVICON'] = 'False'
-# ... etc
-```
-
-#### Timeout Settings
-Use appropriate timeouts for migration tests (45s for init, 60s default).
-
-### Plugin Testing & Code Coverage
-
-**Target: 80-90% coverage** for critical plugins (screenshot, chrome, singlefile, dom)
-
-```bash
-# Run plugin tests with coverage (both Python + JavaScript)
-bash bin/test_plugins.sh screenshot
-
-# View coverage reports
-bash bin/test_plugins.sh --coverage-report
-# Or individual reports:
-coverage report --show-missing --include='archivebox/plugins/*' --omit='*/tests/*'
-```
-
-#### Plugin Test Structure
-
-Tests are **completely isolated** from ArchiveBox - they replicate production directory structure in temp dirs:
-
-```python
-# Correct production paths:
-# Crawl:    DATA_DIR/users/{username}/crawls/YYYYMMDD/example.com/{crawl-id}/{plugin}/
-# Snapshot: DATA_DIR/users/{username}/snapshots/YYYYMMDD/example.com/{snapshot-uuid}/{plugin}/
-
-with tempfile.TemporaryDirectory() as tmpdir:
-    data_dir = Path(tmpdir)
-
-    # Crawl-level plugin (e.g., chrome launcher)
-    crawl_dir = data_dir / 'users' / 'testuser' / 'crawls' / '20240101' / 'example.com' / 'crawl-123'
-    chrome_dir = crawl_dir / 'chrome'
-    chrome_dir.mkdir(parents=True)
-
-    # Snapshot-level plugin (e.g., screenshot)
-    snapshot_dir = data_dir / 'users' / 'testuser' / 'snapshots' / '20240101' / 'example.com' / 'snap-456'
-    screenshot_dir = snapshot_dir / 'screenshot'
-    screenshot_dir.mkdir(parents=True)
-
-    # Run hook in its output directory
-    result = subprocess.run(
-        ['node', str(SCREENSHOT_HOOK), '--url=https://example.com'],
-        cwd=str(screenshot_dir),
-        env={**get_test_env(), 'EXTRA_CONTEXT': '{"snapshot_id":"snap-456"}'},
-        capture_output=True,
-        timeout=120
-    )
-```
-
-#### Coverage Improvement Loop
-
-To improve from ~20% to 80%+:
-
-1. **Run tests**: `bash bin/test_plugins.sh screenshot` → Shows: `19.1% (13/68 ranges)`
-2. **Identify gaps**: Check hook file for untested paths (session connection vs fallback, config branches, error cases)
-3. **Add tests**: Test both execution paths (connect to session + launch own browser), skip conditions, error cases, config variations
-4. **Verify**: Re-run tests → Should show: `85%+ (58+/68 ranges)`
-
-**Critical**: JavaScript hooks have TWO paths that both must be tested (connect to session ~50% + launch browser ~30% + shared ~20%). Testing only one path = max 50% coverage possible!
-
-## Database Migrations
-
-### Generate and Apply Migrations
-```bash
-# Generate migrations (run from archivebox subdirectory)
-cd archivebox
-./manage.py makemigrations
-
-# Apply migrations to test database
-cd data/
-archivebox init
-```
-
-### Schema Versions
-- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
-- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
-- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
-- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
-
-### Testing a Migration Path
-1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
-2. Seed with realistic test data using `seed_0_X_data()`
-3. Run `archivebox init` to trigger migrations
-4. Verify data preservation with `verify_*` functions
-5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
-
-### Squashed Migrations
-When testing 0.8.x (dev branch), you must record ALL replaced migrations:
-```python
-# The squashed migration replaces these - all must be recorded
-('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
-('core', '0024_auto_20240513_1143'),
-# ... all 52 migrations from 0023-0074 ...
-('core', '0023_new_schema'),  # Also record the squashed migration itself
-```
-
-### Migration Strategy
-- Squashed migrations for clean installs
-- Individual migrations recorded for upgrades from dev branch
-- `replaces` attribute in squashed migrations lists what they replace
-
-### Migration Gotchas
-
-#### Circular FK References in Schemas
-SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
-
-## Plugin System Architecture
-
-### Plugin Dependency Rules
-
-Like other plugins, chrome plugins **ARE NOT ALLOWED TO DEPEND ON ARCHIVEBOX OR DJANGO**.
-However, they are allowed to depend on two shared files ONLY:
-- `archivebox/plugins/chrome/chrome_utils.js` ← source of truth API for all basic chrome ops
-- `archivebox/plugins/chrome/tests/chrome_test_utils.py` ← use for your tests, do not implement launching/killing/pid files/cdp/etc. in python, just extend this file as needed.
-
-### Chrome-Dependent Plugins
-
-Many plugins depend on Chrome/Chromium via CDP (Chrome DevTools Protocol). When checking for script name references or debugging Chrome-related issues, check these plugins:
-
-**Main puppeteer-based chrome installer + launcher plugin**:
-- `chrome` - Core Chrome integration (CDP, launch, navigation)
-
-**Metadata extraction using chrome/chrome_utils.js / CDP**:
-- `dns` - DNS resolution info
-- `ssl` - SSL certificate info
-- `headers` - HTTP response headers
-- `redirects` - Capture redirect chains
-- `staticfile` - Direct file downloads (e.g. if the url itself is a .png, .exe, .zip, etc.)
-- `responses` - Capture network responses
-- `consolelog` - Capture console.log output
-- `title` - Extract page title
-- `accessibility` - Extract accessibility tree
-- `seo` - Extract SEO metadata
-
-**Extensions installed using chrome/chrome_utils.js / controlled using CDP**:
-- `ublock` - uBlock Origin ad blocking
-- `istilldontcareaboutcookies` - Cookie banner dismissal
-- `twocaptcha` - 2captcha CAPTCHA solver integration
-
-**Page-alteration plugins to prepare the content for archiving**:
-- `modalcloser` - Modal dialog dismissal
-- `infiniscroll` - Infinite scroll handler
-
-**Main Extractor Outputs**:
-- `dom` - DOM snapshot extraction
-- `pdf` - Generate PDF snapshots
-- `screenshot` - Generate screenshots
-- `singlefile` - SingleFile archival, can be single-file-cli that launches chrome, or singlefile extension running inside chrome
-
-**Crawl URL parsers** (post-process dom.html, singlefile.html, staticfile, responses, headers, etc. for URLs to re-emit as new queued Snapshots during recursive crawling):
-- `parse_dom_outlinks` - Extract outlinks from DOM (special, uses CDP to directly query browser)
-- `parse_html_urls` - Parse URLs from HTML (doesn't use chrome directly, just reads dom.html)
-- `parse_jsonl_urls` - Parse URLs from JSONL (doesn't use chrome directly, just reads dom.html)
-- `parse_netscape_urls` - Parse Netscape bookmark format (doesn't use chrome directly, just reads dom.html)
-
-### Finding Chrome-Dependent Plugins
-
-```bash
-# Find all files containing "chrom" (case-insensitive)
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d: -f1 | sort -u
-
-# Or get just the plugin names
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d/ -f3 | sort -u
-```
-
-**Note**: This list may not be complete. Always run the grep command above when checking for Chrome-related script references or debugging Chrome integration issues.
-
-## Architecture Notes
-
-### Crawl Model (0.9.x)
-- Crawl groups multiple Snapshots from a single `add` command
-- Each `add` creates one Crawl with one or more Snapshots
-- Seed model was removed - crawls now store URLs directly
-
-## Code Coverage
-
-### Overview
-
-Coverage tracking is enabled for passive collection across all contexts:
-- Unit tests (pytest)
-- Integration tests
-- Dev server (manual testing)
-- CLI usage
-
-Coverage data accumulates in `.coverage` file and can be viewed/analyzed to find dead code.
-
-### Install Coverage Tools
-
-```bash
-uv sync --dev  # Installs pytest-cov and coverage
-```
-
-### Running with Coverage
-
-#### Unit Tests
-```bash
-# Run tests with coverage
-pytest --cov=archivebox --cov-report=term archivebox/tests/
-
-# Or run specific test file
-pytest --cov=archivebox --cov-report=term archivebox/tests/test_migrations_08_to_09.py
-```
-
-#### Dev Server with Coverage
-```bash
-# Start dev server with coverage tracking
-coverage run --parallel-mode -m archivebox server
-
-# Or CLI commands
-coverage run --parallel-mode -m archivebox init
-coverage run --parallel-mode -m archivebox add https://example.com
-```
-
-#### Manual Testing (Always-On)
-To enable coverage during ALL Python executions (passive tracking):
-
-```bash
-# Option 1: Use coverage run wrapper
-coverage run --parallel-mode -m archivebox [command]
-
-# Option 2: Set environment variable (tracks everything)
-export COVERAGE_PROCESS_START=pyproject.toml
-# Now all Python processes will track coverage
-archivebox server
-archivebox add https://example.com
-```
-
-### Viewing Coverage
-
-#### Text Report (Quick View)
-```bash
-# Combine all parallel coverage data
-coverage combine
-
-# View summary
-coverage report
-
-# View detailed report with missing lines
-coverage report --show-missing
-
-# View specific file
-coverage report --include="archivebox/core/models.py" --show-missing
-```
-
-#### JSON Report (LLM-Friendly)
-```bash
-# Generate JSON report
-coverage json
-
-# View the JSON
-cat coverage.json | jq '.files | keys'  # List all files
-
-# Find files with low coverage
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 50) | "\(.key): \(.value.summary.percent_covered)%"'
-
-# Find completely uncovered files (dead code candidates)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# Get missing lines for a specific file
-cat coverage.json | jq '.files["archivebox/core/models.py"].missing_lines'
-```
-
-#### HTML Report (Visual)
-```bash
-# Generate interactive HTML report
-coverage html
-
-# Open in browser
-open htmlcov/index.html
-```
-
-### Isolated Runs
-
-To measure coverage for specific scenarios:
-
-```bash
-# 1. Reset coverage data
-coverage erase
-
-# 2. Run your isolated test/scenario
-pytest --cov=archivebox archivebox/tests/test_migrations_fresh.py
-# OR
-coverage run --parallel-mode -m archivebox add https://example.com
-
-# 3. View results
-coverage combine
-coverage report --show-missing
-
-# 4. Optionally export for analysis
-coverage json
-```
-
-### Finding Dead Code
-
-```bash
-# 1. Run comprehensive tests + manual testing to build coverage
-pytest --cov=archivebox archivebox/tests/
-coverage run --parallel-mode -m archivebox server  # Use the app manually
-coverage combine
-
-# 2. Find files with 0% coverage (strong dead code candidates)
-coverage json
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# 3. Find files with <10% coverage (likely dead code)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 10) | "\(.key): \(.value.summary.percent_covered)%"' | sort -t: -k2 -n
-
-# 4. Generate detailed report for analysis
-coverage report --show-missing > coverage_report.txt
-```
-
-### Tips
-
-- **Parallel mode** (`--parallel-mode`): Allows multiple processes to track coverage simultaneously without conflicts
-- **Combine**: Always run `coverage combine` before viewing reports to merge parallel data
-- **Reset**: Use `coverage erase` to start fresh for isolated measurements
-- **Branch coverage**: Enabled by default - tracks if both branches of if/else are executed
-- **Exclude patterns**: Config in `pyproject.toml` excludes tests, migrations, type stubs
-
-## Debugging Tips
-
-### Check Migration State
-```bash
-sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
-```
-
-### Check Table Schema
-```bash
-sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
-```
-
-### Verbose Test Output
-```bash
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
-```
-
-### Kill Zombie Chrome Processes
-```bash
-./bin/kill_chrome.sh
-```

Dockerfile

@@ -1,38 +1,153 @@
-# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
-#     python3.13, uv, python3-ldap
-#     curl, wget, git, dig, ping, tree, nano
-#     node, npm
-#     ArchiveBox and plugin runtime dependencies installed by archivebox init --install
-# Usage:
-#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
-#     docker build . -t archivebox
-#     docker run -v "$PWD/data":/data archivebox init
-#     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
-#     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
-#     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
-#     docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
-# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+# syntax=docker/dockerfile:1.7
 
+# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
+# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
+# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
+# Build abx-dl first, then point this file at it:
+#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
+#       --build-context abxbus=../abxbus \
+#       --build-context abxpkg=../abxpkg \
+#       --build-context abx-plugins=../abx-plugins \
+#       -t archivebox/abx-dl:dev
+#   docker buildx build . -f Dockerfile \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
+#       -t archivebox:multistage
 
-#########################################################################################
+ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
 
-### Example: Using ArchiveBox in your own project's Dockerfile ########
-
-# FROM python:3.13-slim
-# WORKDIR /data
-# RUN pip install archivebox>=0.9.0   # use latest release here
-# RUN archivebox install
-# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
-
-#########################################################################################
+FROM archivebox/sonic:1.4.9 AS sonic
+FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
 
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
-ARG TARGETVARIANT=
+ARG TARGETVARIANT
 
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ubuntu:24.04
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_COMPILE=1 \
+    PIP_ONLY_BINARY=aiohttp \
+    npm_config_loglevel=error
+
+ENV PYTHON_VERSION=3.13 \
+    NODE_VERSION=24
+
+ENV ARCHIVEBOX_USER=archivebox \
+    DEFAULT_ARCHIVEBOX_UID=911 \
+    DEFAULT_ARCHIVEBOX_GID=911 \
+    IN_DOCKER=True
+
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    CONFIG_DIR=/opt/archivebox \
+    LIB_DIR=/opt/archivebox/lib \
+    ABXPKG_LIB_DIR=/opt/archivebox/lib \
+    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
+    PERSONAS_DIR=/data/personas \
+    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
+    CHROME_HEADLESS=true \
+    CHROME_SANDBOX=false \
+    CHROME_ISOLATION=crawl \
+    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
+
+ENV TMP_DIR=/tmp/archivebox \
+    PIP_VENV_PYTHON=/venv/bin/python3 \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no
+
+ENV HOME=/home/archivebox \
+    XDG_CONFIG_HOME=/home/archivebox/.config \
+    XDG_CACHE_HOME=/home/archivebox/.cache \
+    ABXPKG_INSTALL_TIMEOUT=600 \
+    ABXPKG_POSTINSTALL_SCRIPTS=True \
+    ABXPKG_MIN_RELEASE_AGE=0 \
+    TIMEOUT=600
+
+ENV UV_COMPILE_BYTECODE=false \
+    UV_PYTHON_PREFERENCE=managed \
+    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv \
+    VIRTUAL_ENV=/venv \
+    PATH="/venv/bin:/opt/node/bin:$PATH"
+
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
+WORKDIR "$CODE_DIR"
+
+RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
+    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && uname -a \
+    && sed -n '1,7p' /etc/os-release \
+    && which node && node --version \
+    && which uv && uv self version \
+    ) | tee -a /VERSION.txt
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends ripgrep \
+    && rm -rf /var/lib/apt/lists/*
+
+FROM archivebox-runtime-base AS archivebox-builder
+
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
+    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
+    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
+    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
+    && rm -f /etc/apt/apt.conf.d/docker-clean \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
+    && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \
+    && /usr/bin/uv pip install setuptools pip wheel \
+    && /usr/bin/uv sync \
+        --refresh \
+        --no-dev \
+        --inexact \
+        --no-install-project \
+        --no-install-workspace \
+        --no-sources \
+    && (find /venv/lib/python3.*/site-packages -type f -name '*.so' -exec strip --strip-unneeded {} + 2>/dev/null || true) \
+    && rm -f /venv/bin/uv /venv/bin/uvx \
+    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && COMMIT_HASH="$( \
+        if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
+            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
+            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+                echo "$HEAD_REF"; \
+            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
+                REF_PATH="${HEAD_REF#ref: }"; \
+                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
+            fi; \
+        fi)" \
+    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
+    && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
+    && rm -f /venv/bin/uv /venv/bin/uvx \
+    && (/usr/bin/uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
+    && rm -rf "$CODE_DIR/.git"
+
+FROM archivebox-runtime-base
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -43,369 +158,74 @@ LABEL name="archivebox" \
     org.opencontainers.image.vendor="ArchiveBox" \
     org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
     org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile" \
-    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
-    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
-    com.docker.desktop.extension.api.version=">= 1.4.7" \
-    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
-    com.docker.extension.publisher-url="https://archivebox.io" \
-    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
-    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
-    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
-    com.docker.extension.categories='database,utility-tools'
+    com.docker.image.source.entrypoint="Dockerfile"
 
-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-######### Environment Variables #################################
-
-# Global build-time and runtime environment constants + default pkg manager config
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_ONLY_BINARY=aiohttp \
-    npm_config_loglevel=error
-
-# Language Version config
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22.22.3
-
-# Non-root User config
-ENV ARCHIVEBOX_USER="archivebox" \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
-
-# ArchiveBox Source Code + Lib + Data paths
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    SONIC_BINARY=/opt/archivebox/lib/env/bin/sonic \
-    PLAYWRIGHT_BROWSERS_PATH=/browsers
-
-# Bash SHELL config
-# http://redsymbol.net/articles/unofficial-bash-strict-mode/
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
-
-######### System Environment ####################################
-
-# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
-WORKDIR "$CODE_DIR"
-
-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-# Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && echo \
-    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
-    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
-    && echo \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && which bash && bash --version | sed -n '1p' \
-    && which dpkg && dpkg --version | sed -n '1p' \
-    && echo -e '\n\n' && env && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-# Create non-privileged user for archivebox and chrome
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system $ARCHIVEBOX_USER \
-    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
-    | tee -a /VERSION.txt
-    # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
-    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
-
-# Install system apt dependencies (adding backports to access more recent apt updates)
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
-    && mkdir -p /etc/apt/keyrings \
-    && apt-get update -qq \
-    && apt-get install -qq -y \
-        # 1. packaging dependencies
-        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
-        # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep dnsutils git ripgrep python3.12-venv default-jre-headless \
-        # 3. frivolous CLI helpers to make debugging failed archiving easier
-        tree nano iputils-ping \
-        # nano iputils-ping dnsutils htop procps jq yq
-    && rm -rf /var/lib/apt/lists/*
-
-# Install sonic search backend
 COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-RUN (which sonic && sonic --version) | tee -a /VERSION.txt
 
-######### Language Environments ####################################
+COPY --from=archivebox-builder /opt/uv/python /opt/uv/python
+COPY --from=archivebox-builder /venv /venv
+COPY --from=archivebox-builder /app /app
+COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
 
-# Set up Python environment
-# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
-#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
-    # && apt-get update -qq \
-    # && apt-get install -qq -y --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
-    # && rm -rf /var/lib/apt/lists/* \
-    # tell PDM to allow using global system python site packages
-    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
-    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
-    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
-    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
-    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
-    # && python3 -m venv $GLOBAL_VENV \
-    # install global dependencies / python build dependencies in GLOBAL_VENV
-    # && pip install --upgrade pip setuptools wheel \
-    # Save version info
-    # && ( \
-    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
-    #     && which pip && pip --version \
-    #     # && which pdm && pdm --version \
-    #     && echo -e '\n\n' \
-    # ) | tee -a /VERSION.txt
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
+    && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
+    && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
+    && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
+    && ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
+    && ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
+    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
+    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
+    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
+    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
+    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
+    && (which sonic && sonic --version) | tee -a /VERSION.txt \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.cache" \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER/.cache/abxbus/semaphores" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \
+    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/abxbus/semaphores" \
+    && openssl rand -hex 16 > /etc/machine-id \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
+    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
 
-
-# Set up Node environment from the official platform tarball. This avoids
-# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into
-# emulated Docker builds.
-RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    case "$TARGETARCH" in \
-        amd64) NODE_DIST_ARCH="x64" ;; \
-        arm64) NODE_DIST_ARCH="arm64" ;; \
-        *) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \
-    esac \
-    && NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \
-    && echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \
-    && grep "  ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \
-    && tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \
-    && rm "$NODE_TARBALL" SHASUMS256.txt \
-    # Save version info
-    && ( \
-        which node && node --version \
-        && which npm && npm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-# Set up uv and main app /venv
-RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
-ENV UV_COMPILE_BYTECODE=0 \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv
-WORKDIR "$CODE_DIR"
-# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
-    && uv venv /venv --python ${PYTHON_VERSION}
-ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
-RUN uv pip install setuptools pip \
-    && ( \
-        which python3 && python3 --version \
-        && which uv && uv self version \
-        && uv python find --system && uv python find \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-######### ArchiveBox & Extractor Dependencies ##################################
-
-# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc \
-        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
-        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-    && uv pip install \
-        "python-ldap>=3.4.3" \
-    && apt-get purge -y \
-        python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
-
-
-# Runtime config used by plugin hooks. Plugin binaries and npm packages are
-# installed into LIB_DIR below by archivebox init --install and resolved from
-# LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH.
-ENV PERSONAS_DIR=/data/personas \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-######### Build Dependencies ####################################
-
-
-# Install ArchiveBox Python venv dependencies from pyproject.toml.
-RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends build-essential gcc python3-dev \
-    && uv --no-cache sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /var/lib/apt/lists/*
-    # installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies
-
-# Setup ArchiveBox runtime config
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/usr/bin/python3.12 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-WORKDIR "$DATA_DIR"
-RUN openssl rand -hex 16 > /etc/machine-id \
-    && mkdir -p "$DATA_DIR" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" \
-    && mkdir -p "$TMP_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
-    && mkdir -p "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-# Pre-bake plugin-managed runtime dependencies using the same abx-dl installer
-# path users run later, before copying ArchiveBox source so source-only edits do
-# not invalidate the heavy browser/plugin dependency layer.
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
-    && export PERSONAS_DIR="$LIB_DIR/personas" \
-    && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
-    && export ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 \
-    && mkdir -p "$LIB_DIR" \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends build-essential tesseract-ocr tesseract-ocr-eng \
-    && abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" gallery-dl \
-    && abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" --overrides='{"pip":{"install_args":["--no-deps","forum-dl","chardet==5.2.0","pydantic==2.12.3","pydantic-core==2.41.4","typing-extensions>=4.14.1","annotated-types>=0.6.0","typing-inspection>=0.4.2","beautifulsoup4","soupsieve","lxml","requests","urllib3","certifi","idna","charset-normalizer","tenacity","python-dateutil","six","html2text","warcio"]}}' forum-dl \
-    && if [ "$TARGETARCH" = "arm64" ]; then \
-        abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \
-        abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
-    fi \
-    && ABXPKG_INSTALL_TIMEOUT=600 TIMEOUT=600 PUID=0 PGID=0 abx-dl plugins --install \
-        accessibility archivedotorg archivewebpage base chrome chrome_mhtml chrome_screencast \
-        claudechrome claudecode claudecodecleanup claudecodeextract consolelog defuddle dns dom \
-        favicon forumdl gallerydl git hashes headers htmltotext infiniscroll \
-        istilldontcareaboutcookies liteparse media mercury modalcloser opendataloader papersdl \
-        parse_dom_outlinks parse_html_urls parse_jsonl_urls parse_netscape_urls parse_rss_urls \
-        parse_txt_urls pdf readability redirects responses screenshot search_backend_ripgrep \
-        search_backend_sonic search_backend_sqlite seo singlefile ssl sslcerts staticfile title \
-        trafilatura ublock wget ytdlp \
-    && abxpkg install --no-cache --binproviders=chromewebstore --overrides='{"chromewebstore":{"install_args":["fpeoodllldobpkbkabpblcfaogecpndd","--name=archivewebpage"]}}' archivewebpage \
-    && test -f "$LIB_DIR/chromewebstore/extensions/fpeoodllldobpkbkabpblcfaogecpndd__archivewebpage/manifest.json" \
-    && mkdir -p "$LIB_DIR/env/bin" \
-    && ln -sf "$(command -v node)" "$LIB_DIR/env/bin/node" \
-    && ln -sf "$(command -v npm)" "$LIB_DIR/env/bin/npm" \
-    && ln -sf "$(command -v java)" "$LIB_DIR/env/bin/java" \
-    && ln -sf "$(command -v git)" "$LIB_DIR/env/bin/git" \
-    && ln -sf "$(command -v rg)" "$LIB_DIR/env/bin/rg" \
-    && ln -sf "$(command -v sonic)" "$LIB_DIR/env/bin/sonic" \
-    && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \
-    && mkdir -p /opt/archivebox/lib-layer \
-    && cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \
-    && apt-get purge -y build-essential \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/* \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer
-
-RUN rm -rf "$LIB_DIR" \
-    && mv /opt/archivebox/lib-layer "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"
-
-# Install ArchiveBox Python package from the checked-out source.
-WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && pip install \
-        --no-deps \
-        "$CODE_DIR" \
-    && ( \
-        pip show archivebox \
-        && which archivebox \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
-    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
-
-# Initialize an empty image collection without rerunning dependency installs.
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
     && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
-    && PUID=0 PGID=0 archivebox init \
-    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
+    && archivebox init \
+    && (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
         "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
         "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
         "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
         "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        2>/dev/null || true)
+        "$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
+        "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" "/home/$ARCHIVEBOX_USER/.cache" \
+        2>/dev/null || true) \
+    && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
 
-# Print version for nice docker finish summary
-RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+RUN chmod +x "$CODE_DIR"/bin/*.sh \
+    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+
+RUN "$LIB_DIR/playwright/bin/chromium" --version | tee -a /VERSION.txt \
+    && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
+    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
+    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
+    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
+    && for forbidden_bin in gcc g++ make; do ! command -v "$forbidden_bin" || (echo "Unexpected build tool in runtime: $forbidden_bin=$(command -v "$forbidden_bin")" >&2 && exit 1); done \
+    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
+    && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER/.cache" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "/home/$ARCHIVEBOX_USER/.cache/abxbus/semaphores" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "/home/$ARCHIVEBOX_USER/.cache/uv" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install \
+    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+
+RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
     && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
     ) | tee -a /VERSION.txt
 
-# Verify ArchiveBox is installed and write full version/dependency info.
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
-    && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury papersdl parse_rss_urls readability search_backend_sonic opendataloader search_backend_ripgrep 2>&1 | tee -a /VERSION.txt \
-    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-####################################################
-
-# Expose ArchiveBox's main interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000

Dockerfile.multistage

@@ -1,229 +0,0 @@
-# syntax=docker/dockerfile:1.7
-
-# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
-# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
-# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
-# Build abx-dl first, then point this file at it:
-#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
-#       --build-context abxbus=../abxbus \
-#       --build-context abxpkg=../abxpkg \
-#       --build-context abx-plugins=../abx-plugins \
-#       -t archivebox/abx-dl:dev
-#   docker buildx build . -f Dockerfile.multistage \
-#       --build-context abx-dl=docker-image://archivebox/abx-dl:dev \
-#       --build-arg ABX_DL_IMAGE=abx-dl \
-#       -t archivebox:multistage
-
-ARG TARGETPLATFORM=linux/amd64
-ARG TARGETOS=linux
-ARG TARGETARCH=amd64
-ARG TARGETVARIANT=
-ARG ABX_DL_IMAGE=abx-dl
-
-FROM ${ABX_DL_IMAGE} AS abx-dl
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ubuntu:24.04 AS archivebox-runtime-base
-
-ARG TARGETPLATFORM=linux/amd64
-ARG TARGETOS=linux
-ARG TARGETARCH=amd64
-ARG TARGETVARIANT
-
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_ONLY_BINARY=aiohttp \
-    npm_config_loglevel=error
-
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22.22.3
-
-ENV ARCHIVEBOX_USER=archivebox \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
-
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
-    PERSONAS_DIR=/data/personas \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/venv/bin/python3 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-ENV UV_COMPILE_BYTECODE=0 \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv \
-    VIRTUAL_ENV=/venv \
-    PATH="/venv/bin:/opt/node/bin:$PATH"
-
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
-WORKDIR "$CODE_DIR"
-
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing ArchiveBox base runtime dependencies for $TARGETPLATFORM..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y \
-        apt-transport-https apt-utils ca-certificates curl wget gnupg2 \
-        dumb-init gosu unzip git grep ripgrep dnsutils iputils-ping procps tree nano \
-        cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \
-        libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \
-        libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \
-        libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 \
-        libxext6 libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1 \
-        fonts-liberation fonts-noto-color-emoji xdg-utils \
-        ffmpeg imagemagick tesseract-ocr tesseract-ocr-eng openjdk-21-jre-headless \
-    && rm -rf /var/lib/apt/lists/*
-
-# Runtime-owned layers copied from the abx-dl image.
-COPY --from=abx-dl /bin/uv /bin/uv
-COPY --from=abx-dl /opt/uv/python /opt/uv/python
-COPY --from=abx-dl /opt/node /opt/node
-COPY --from=abx-dl /venv /venv
-COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt
-
-RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && which python3 && python3 --version \
-    && which node && node --version \
-    && which uv && uv self version \
-    ) | tee -a /VERSION.txt
-
-ENV PYTHONDONTWRITEBYTECODE=1
-
-FROM archivebox-runtime-base AS archivebox-builder
-
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && uv sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && apt-get autoremove -y \
-    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /var/lib/apt/lists/*
-
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && uv pip install --no-deps "$CODE_DIR" \
-    && (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
-
-FROM archivebox-runtime-base
-
-LABEL name="archivebox" \
-    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
-    description="All-in-one self-hosted internet archiving solution" \
-    homepage="https://github.com/ArchiveBox/ArchiveBox" \
-    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
-    org.opencontainers.image.title="ArchiveBox" \
-    org.opencontainers.image.vendor="ArchiveBox" \
-    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
-    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile.multistage"
-
-COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
-COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-
-COPY --from=archivebox-builder /venv /venv
-COPY --from=archivebox-builder /app /app
-COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
-COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib
-
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system "$ARCHIVEBOX_USER" \
-    && useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && (which sonic && sonic --version) | tee -a /VERSION.txt \
-    && mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && openssl rand -hex 16 > /etc/machine-id \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
-    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-WORKDIR "$DATA_DIR"
-RUN echo "[+] Initializing image collection..." \
-    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
-    && PUID=0 PGID=0 archivebox init \
-    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
-        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
-        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
-        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
-        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        2>/dev/null || true)
-
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
-    && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
-    && "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
-    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
-    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
-    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
-    && ! command -v gcc \
-    && ! command -v g++ \
-    && ! command -v make \
-    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
-    ) | tee -a /VERSION.txt
-
-WORKDIR "$DATA_DIR"
-VOLUME "$DATA_DIR"
-EXPOSE 8000
-
-HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
-
-ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Nick Sweeting
+Copyright (c) 2026 Nick Sweeting
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -295,18 +295,18 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
 <ol>
-<li>Download and install the <code>.deb</code> package from the <a href="https://github.com/ArchiveBox/ArchiveBox/releases">latest release</a>.
-<pre lang="bash"><code style="white-space: pre-line"># download the .deb for your architecture (amd64 or arm64)
-ARCH="$(dpkg --print-architecture)"
-VERSION="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'].lstrip('v'))")"
-curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${VERSION}_${ARCH}.deb" -o /tmp/archivebox.deb
-sudo apt install /tmp/archivebox.deb
-archivebox version                         # make sure all dependencies are installed
+<li>Add the third-party ArchiveBox apt repo and install <code>archivebox</code>.
+<pre lang="bash"><code style="white-space: pre-line">echo 'deb [trusted=yes] https://archivebox.github.io/debian-archivebox dev main' | sudo tee /etc/apt/sources.list.d/archivebox.list
+sudo apt update
+sudo apt install archivebox
+archivebox version                         # make sure the package is installed
 </code></pre>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --install
+archivebox init
+sudo archivebox install
+archivebox add 'https://example.com'
 </code></pre>
 <br/>
 </li>
@@ -476,6 +476,22 @@ For more discussion on managed and paid hosting options see here: <a href="https
 ArchiveBox commands can be run in a terminal [directly on your host](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), or via [Docker](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1)/[Docker Compose](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage).  
 <sup>(depending on how you chose to install it above)</sup>
 
+<!--
+```bash
+set -euo pipefail
+__archivebox_docs_home="$(mktemp -d)"
+export HOME="$__archivebox_docs_home"
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init
+archivebox version
+archivebox help
+test -f index.sqlite3
+test -d archive
+rm -rf "$__archivebox_docs_home"
+```
+-->
+<!--pytest.mark.skip(reason='Mixed host/Docker alternatives include interactive Docker commands')-->
 ```bash
 mkdir -p ~/archivebox/data   # create a new data dir anywhere
 cd ~/archivebox/data         # IMPORTANT: cd into the directory
@@ -512,10 +528,12 @@ archivebox persona create --import=chrome personal
 # make sure you have pip-installed ArchiveBox and it's available in your $PATH first  
 <br/>
 # archivebox [subcommand] [--help]
-archivebox init --install      # safe to run init multiple times (also how you update versions)
+mkdir -p ~/archivebox/data && cd ~/archivebox/data
+archivebox init
+sudo archivebox install
+archivebox add 'https://example.com'
 archivebox version           # get archivebox version info + check dependencies
 archivebox help              # get list of archivebox subcommands that can be run
-archivebox add --depth=1 'https://news.ycombinator.com'
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage">Usage: CLI Usage</a> wiki. ➡️</i>
 </details>
@@ -529,10 +547,11 @@ archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
 # docker compose run archivebox [subcommand] [--help]
-docker compose run archivebox init --install
+docker compose run archivebox init
+docker compose run archivebox install
 docker compose run archivebox version
 docker compose run archivebox help
-docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
+docker compose run archivebox add 'https://example.com'
 # to start webserver: docker compose up
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage">Usage: Docker Compose CLI</a> wiki. ➡️</i>
@@ -547,10 +566,11 @@ docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you create and cd into in a new empty directory first  
 <br/>
 # docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox init --install
+docker run -v $PWD:/data -it archivebox/archivebox init
+docker run -v $PWD:/data -it archivebox/archivebox install
 docker run -v $PWD:/data -it archivebox/archivebox version
 docker run -v $PWD:/data -it archivebox/archivebox help
-docker run -v $PWD:/data -it archivebox/archivebox add --depth=1 'https://news.ycombinator.com'
+docker run -v $PWD:/data -it archivebox/archivebox add 'https://example.com'
 # to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
@@ -676,10 +696,36 @@ docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://exampl
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e1e5bd78-b0b6-45dc-914c-e1046fee4bc4" width="330px" align="right" style="float: right"/>
 
 
+<!--
+```bash
+set -euo pipefail
+__archivebox_docs_home="$(mktemp -d)"
+export HOME="$__archivebox_docs_home"
+mkdir -p ~/archivebox/data ~/Downloads
+cd ~/archivebox/data
+archivebox init
+cat > ~/Downloads/some_feed.xml <<'EOF'
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>ArchiveBox docs test feed</title>
+    <link>https://example.com/</link>
+    <description>ArchiveBox docs test feed</description>
+    <item>
+      <title>Feed item</title>
+      <link>https://example.com/from-feed</link>
+      <guid>https://example.com/from-feed</guid>
+    </item>
+  </channel>
+</rss>
+EOF
+```
+-->
+<!--pytest-codeblocks:cont-->
 ```bash
 # archivebox add --help
 archivebox add 'https://example.com/some/page'
-archivebox add --parser=generic_rss < ~/Downloads/some_feed.xml
+archivebox add --depth=1 --plugins=parse_rss_urls "file://$HOME/Downloads/some_feed.xml"
 archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
 echo 'http://example.com' | archivebox add
 echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add
@@ -689,6 +735,19 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
 # if using Docker Compose, add -T when piping stdin / stdout:
 # echo 'https://example.com' | docker compose run -T archivebox add
 ```
+<!--pytest-codeblocks:cont-->
+<!--
+```bash
+archivebox list --json > snapshots.json
+grep -q 'https://example.com/some/page' snapshots.json
+grep -q 'https://example.com/from-feed' snapshots.json
+grep -q 'https://news.ycombinator.com#2020-12-12' snapshots.json
+grep -q 'http://example.com' snapshots.json
+test -d archive
+test "$(find archive -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" -ge 4
+rm -rf "$__archivebox_docs_home"
+```
+-->
 
 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
 
@@ -1085,7 +1144,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
 </li>
 <li><strong>Try to keep the <code>data/index.sqlite3</code> file on local drive (not a network mount)</strong> or SSD for maximum performance, however the <code>data/archive/</code> folder can be on a network mount or slower HDD.</li>
-<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, you may need to set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid"><code>PUID</code> & <code>PGID</code></a> and <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304">disable <code>root_squash</code></a> on your fileshare server.
+<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, make sure the mounted data directory is writable by its intended owner and consider disabling <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304"><code>root_squash</code></a> on your fileshare server.
 </li>
 </ul>
 
@@ -1095,7 +1154,6 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid">Wiki: Configuration (<code>PUID</code> & <code>GUID</code>)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root">Wiki: Security Overview (Do Not Run as Root)</a></li>
 </ul>
 
@@ -1348,6 +1406,7 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc
 
 First make sure you have `uv` installed: https://docs.astral.sh/uv/getting-started/installation/
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 git clone https://github.com/ArchiveBox/monorepo
 cd monorepo
@@ -1369,6 +1428,7 @@ Repos included in monorepo setup:
 
 #### 2. Option A: Install the Python, JS, and system dependencies directly on your machine
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # Install ArchiveBox runtime dependencies
 mkdir -p data && cd data
@@ -1384,6 +1444,7 @@ archivebox server 0.0.0.0:8000
 
 #### 2. Option B: Build the docker container and use that for development instead
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # Optional: develop via docker by mounting the code dir into the container
 # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
@@ -1413,6 +1474,7 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # set up persistent DEBUG=True for all runs
 archivebox config --set DEBUG=True
@@ -1437,6 +1499,7 @@ https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-dj
 
 If you're looking for the latest `dev` Docker image, it's often available pre-built on Docker Hub, simply pull and use `archivebox/archivebox:dev`.
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 docker pull archivebox/archivebox:dev
 docker run archivebox/archivebox:dev version
@@ -1444,9 +1507,10 @@ docker run archivebox/archivebox:dev version
 ```
 
 ##### Build Branch from Source
-  
+
 You can also build and run any branch yourself from source, for example to build & use `dev` locally:
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # docker-compose.yml:
 services:
@@ -1471,6 +1535,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/lint.sh
 ./bin/test.sh
@@ -1484,6 +1549,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # generate the database migrations after changes to models.py
 cd archivebox/
@@ -1548,6 +1614,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/build.sh
 
@@ -1564,6 +1631,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/release.sh
 

archivebox/__main__.py

@@ -17,4 +17,4 @@ ASCII_LOGO_MINI = r"""
 """
 
 if __name__ == "__main__":
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main(args=sys.argv[1:])

archivebox/api/auth.py

@@ -51,6 +51,27 @@ def auth_using_token(token: str | None, request: HttpRequest | None = None) -> U
     return user
 
 
+def token_from_request(request: HttpRequest) -> str:
+    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
+    auth_header = request.headers.get("Authorization", "")
+    if not token and auth_header.lower().startswith("bearer "):
+        token = auth_header.split(None, 1)[1].strip()
+    return token
+
+
+def authenticated_user_from_request(request: HttpRequest) -> User | None:
+    user = request.user
+    if user.is_authenticated and user.is_active:
+        return user
+
+    token = token_from_request(request)
+    token_user = auth_using_token(token=token, request=request) if token else None
+    if token_user and token_user.is_active:
+        request.user = token_user
+        return token_user
+    return None
+
+
 def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
     """Given a username and password, check if they are valid and return the corresponding user"""
     user: User | None = None

archivebox/api/v1_cli.py

@@ -8,8 +8,11 @@ from enum import Enum
 from django.http import HttpRequest
 
 from ninja import Router, Schema
+from ninja.errors import HttpError
+from pydantic import Field
 
 from archivebox.misc.util import ansi_to_html
+from archivebox.core.models import SnapshotQuerySet
 
 
 # from .auth import API_AUTH_METHODS
@@ -21,6 +24,7 @@ router = Router(tags=["ArchiveBox CLI Sub-Commands"])
 # Schemas
 
 JSONType = list[Any] | dict[str, Any] | bool | int | str | None
+FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]
 
 
 class CLICommandResponseSchema(Schema):
@@ -32,26 +36,11 @@ class CLICommandResponseSchema(Schema):
     stderr: str
 
 
-class FilterTypeChoices(str, Enum):
-    exact = "exact"
-    substring = "substring"
-    regex = "regex"
-    domain = "domain"
-    tag = "tag"
-    timestamp = "timestamp"
-
-
-class StatusChoices(str, Enum):
-    indexed = "indexed"
-    archived = "archived"
-    unarchived = "unarchived"
-    present = "present"
-    valid = "valid"
-    invalid = "invalid"
-    duplicate = "duplicate"
-    orphaned = "orphaned"
-    corrupted = "corrupted"
-    unrecognized = "unrecognized"
+FilterTypeChoices = Enum(
+    "FilterTypeChoices",
+    {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
+    type=str,
+)
 
 
 class AddCommandSchema(Schema):
@@ -71,14 +60,27 @@ class AddCommandSchema(Schema):
     index_only: bool = False
 
 
-class UpdateCommandSchema(Schema):
-    resume: str | None = None
+class SnapshotFilterCommandSchema(Schema):
     after: float | None = 0
-    before: float | None = 999999999999999
+    before: float | None = None
     filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = ["https://example.com"]
+    filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
+    status: str | None = None
+    url__icontains: str | None = None
+    url__istartswith: str | None = None
+    tag: str | None = None
+    crawl_id: str | None = None
+    limit: int | None = None
+    sort: str | None = None
+    search: str | None = None
+
+
+class UpdateCommandSchema(SnapshotFilterCommandSchema):
+    resume: str | None = None
     batch_size: int = 100
     continuous: bool = False
+    index_only: bool = False
+    migrate_only: bool = False
 
 
 class ScheduleCommandSchema(Schema):
@@ -97,24 +99,23 @@ class ScheduleCommandSchema(Schema):
     clear: bool = False
 
 
-class ListCommandSchema(Schema):
-    filter_patterns: list[str] | None = ["https://example.com"]
-    filter_type: str = FilterTypeChoices.substring
-    status: StatusChoices = StatusChoices.indexed
-    after: float | None = 0
-    before: float | None = 999999999999999
-    sort: str = "bookmarked_at"
+class ListCommandSchema(SnapshotFilterCommandSchema):
     as_json: bool = True
     as_html: bool = False
     as_csv: str | None = "timestamp,url"
     with_headers: bool = False
 
 
-class RemoveCommandSchema(Schema):
-    after: float | None = 0
-    before: float | None = 999999999999999
+class RemoveCommandSchema(SnapshotFilterCommandSchema):
     filter_type: str = FilterTypeChoices.exact
-    filter_patterns: list[str] | None = ["https://example.com"]
+    timeout: float = 60.0
+
+
+def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
+    kwargs = args.dict()
+    kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
+    kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
+    return kwargs
 
 
 @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
@@ -165,24 +166,39 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
 
 @router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
 def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import update
+    from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
+    from archivebox.core.snapshot_status import normalize_snapshot_status
 
-    result = update(
-        filter_patterns=args.filter_patterns or [],
-        filter_type=args.filter_type or FilterTypeChoices.substring,
-        after=args.after,
-        before=args.before,
-        resume=args.resume,
-        batch_size=args.batch_size,
-        continuous=args.continuous,
-        stop_daemon_stack=False,
+    try:
+        status = normalize_snapshot_status(args.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+
+    update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    update_kwargs["status"] = status
+    update_kwargs["stop_daemon_stack"] = False
+
+    is_filtered_update = any(
+        (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
     )
+    matched_snapshot_ids = []
+    if is_filtered_update:
+        matched_snapshot_ids = [
+            str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
+        ]
+
+    update(**update_kwargs)
     stdout = request.__dict__.get("stdout")
     stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
-        "result": result,
+        "result": {
+            "matched_count": len(matched_snapshot_ids),
+            "snapshot_ids": matched_snapshot_ids,
+        }
+        if is_filtered_update
+        else None,
         "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
         "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
     }
@@ -225,29 +241,35 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
 
 @router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
 def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_search import search
+    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
-    result = search(
-        filter_patterns=args.filter_patterns,
-        filter_type=args.filter_type,
-        status=args.status,
-        after=args.after,
-        before=args.before,
-        sort=args.sort,
-        csv=args.as_csv,
-        json=args.as_json,
-        html=args.as_html,
-        with_headers=args.with_headers,
-    )
+    search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    as_json = search_kwargs.pop("as_json")
+    as_html = search_kwargs.pop("as_html")
+    as_csv = search_kwargs.pop("as_csv")
+    with_headers = search_kwargs.pop("with_headers")
+    try:
+        snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
 
     result_format = "txt"
-    if args.as_json:
+    if as_json:
         result_format = "json"
-        result = json.loads(result)
-    elif args.as_html:
+        result = [
+            json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
+            for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
+        ]
+    elif as_html:
         result_format = "html"
-    elif args.as_csv:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
+    elif as_csv:
         result_format = "csv"
+        cols = [col.strip() for col in as_csv.split(",") if col.strip()]
+        rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
+        result = "\n".join((",".join(cols), *rows) if with_headers else rows)
+    else:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
 
     stdout = request.__dict__.get("stdout")
     stderr = request.__dict__.get("stderr")
@@ -264,37 +286,26 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
 @router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
 def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
     from archivebox.cli.archivebox_remove import remove
-    from archivebox.cli.archivebox_search import get_snapshots
     from archivebox.core.models import Snapshot
 
-    filter_patterns = args.filter_patterns or []
-    snapshots_to_remove = get_snapshots(
-        filter_patterns=filter_patterns,
-        filter_type=args.filter_type,
-        after=args.after,
-        before=args.before,
-    )
-    removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)]
+    remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
+    timeout_arg = remove_kwargs.pop("timeout")
+    timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
+    try:
+        snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
 
-    remove(
+    result = remove(
         yes=True,  # no way to interactively ask for confirmation via API, so we force yes
         snapshots=snapshots_to_remove,
-        before=args.before,
-        after=args.after,
-        filter_type=args.filter_type,
-        filter_patterns=filter_patterns,
+        timeout=timeout,
     )
-
-    result = {
-        "removed_count": len(removed_snapshot_ids),
-        "removed_snapshot_ids": removed_snapshot_ids,
-        "remaining_snapshots": Snapshot.objects.count(),
-    }
     stdout = request.__dict__.get("stdout")
     stderr = request.__dict__.get("stderr")
     return {
-        "success": True,
-        "errors": [],
+        "success": bool(result["success"]),
+        "errors": [str(result["error"])] if result["error"] else [],
         "result": result,
         "result_format": "json",
         "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",

archivebox/api/v1_core.py

@@ -29,7 +29,7 @@ from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot, ArchiveResult, Tag
 from archivebox.core.permissions import public_snapshots_queryset
-from archivebox.api.auth import auth_using_token
+from archivebox.api.auth import authenticated_user_from_request
 from archivebox.config.common import get_config
 from archivebox.core.routes_util import build_web_url
 from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
@@ -49,9 +49,10 @@ from archivebox.core.tag_util import (
     rename_tag as rename_tag_record,
 )
 from archivebox.crawls.models import Crawl
-from archivebox.api.v1_crawls import CrawlSchema
+from archivebox.api.v1_crawls import CrawlSchema, get_crawl_by_ref
 from archivebox.search.config import get_search_mode, get_search_mode_backend
 from archivebox.search.query import apply_snapshot_search
+from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status
 
 
 router = Router(tags=["Core Models"])
@@ -79,7 +80,7 @@ class CustomPagination(PaginationBase):
     def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
         limit = min(pagination.limit, 500)
         offset = pagination.offset or (pagination.page * limit)
-        total = queryset.count()
+        total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count()
         total_pages = math.ceil(total / limit)
         current_page = math.ceil(offset / (limit + 1))
         items = queryset[offset : offset + limit]
@@ -212,7 +213,10 @@ class ArchiveResultFilterSchema(FilterSchema):
 @paginate(CustomPagination)
 def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
     """List all ArchiveResult entries matching these filters."""
-    return filters.filter(ArchiveResult.objects.all()).distinct()
+    queryset = filters.filter(ArchiveResult.objects.all())
+    if filters.search or filters.snapshot_tag:
+        return queryset.distinct()
+    return queryset
 
 
 def _uuid_ref_query(field_name: str, ref: str) -> Q:
@@ -854,6 +858,7 @@ class SnapshotFilterSchema(FilterSchema):
     modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
     search: str | None = None
     search_mode: str | None = None
+    status: str | None = None
     url: Annotated[str | None, FilterLookup("url")] = None
     tag: Annotated[str | None, FilterLookup("tags__name")] = None
     title: Annotated[str | None, FilterLookup("title__icontains")] = None
@@ -867,13 +872,20 @@ class SnapshotFilterSchema(FilterSchema):
     def filter_search_mode(self, value: str | None) -> Q:
         return Q()
 
+    def filter_status(self, value: str | None) -> Q:
+        return Q()
+
 
 @router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
 @paginate(CustomPagination)
 def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
     """List all Snapshot entries matching these filters."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = filters.filter(Snapshot.objects.all()).distinct()
+    try:
+        queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+    queryset = filters.filter(queryset).distinct()
     query = (filters.search or "").strip()
     if not query:
         return queryset
@@ -916,18 +928,16 @@ def get_snapshots_rss(
 def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
     """Get a specific Snapshot by id."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.all()
-    try:
-        return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        return queryset.get(_uuid_ref_query("id", snapshot_id))
+    return _get_snapshot_by_ref(snapshot_id)
 
 
 @router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
 def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     tags = normalize_tag_list(data.tags)
-    if data.status is not None and data.status not in Snapshot.StatusChoices.values:
-        raise HttpError(400, f"Invalid status: {data.status}")
+    try:
+        status = normalize_snapshot_status(data.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
     if not data.url.strip():
         raise HttpError(400, "URL is required")
     try:
@@ -938,7 +948,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
         raise HttpError(400, "depth must be between 0 and 4")
 
     if data.crawl_id:
-        crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), data.crawl_id).get()
+        crawl = get_crawl_by_ref(data.crawl_id)
         crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
         tags = tags or crawl_tags
     else:
@@ -955,7 +965,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
         "depth": data.depth,
         "title": data.title,
         "timestamp": str(timezone.now().timestamp()),
-        "status": data.status or Snapshot.StatusChoices.QUEUED,
+        "status": status or Snapshot.StatusChoices.QUEUED,
         "retry_at": timezone.now(),
     }
     snapshot, _ = Snapshot.objects.get_or_create(
@@ -968,10 +978,8 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     if data.title is not None and snapshot.title != data.title:
         snapshot.title = data.title
         update_fields.append("title")
-    if data.status is not None and snapshot.status != data.status:
-        if data.status not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {data.status}")
-        snapshot.status = data.status
+    if status is not None and snapshot.status != status:
+        snapshot.status = status
         update_fields.append("status")
     if update_fields:
         update_fields.append("modified_at")
@@ -992,10 +1000,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
 @router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
 def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
     """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
-    try:
-        snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        snapshot = filter_queryset_by_uuid_substring(Snapshot.objects.all(), snapshot_id).get()
+    snapshot = _get_snapshot_by_ref(snapshot_id)
 
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
@@ -1018,9 +1023,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
         raise HttpError(400, f"Invalid action: {action}")
 
     if "status" in payload:
-        if payload["status"] not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        snapshot.status = payload["status"]
+        try:
+            snapshot.status = normalize_snapshot_status(payload["status"])
+        except ValueError as err:
+            raise HttpError(400, str(err)) from err
         if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
             snapshot.retry_at = None
         update_fields.append("status")
@@ -1289,16 +1295,7 @@ def _public_tag_listing_enabled() -> bool:
 
 
 def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
-    user = request.user
-    if user.is_authenticated:
-        return True
-
-    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-    auth_header = request.headers.get("Authorization", "")
-    if not token and auth_header.lower().startswith("bearer "):
-        token = auth_header.split(None, 1)[1].strip()
-
-    if token and auth_using_token(token=token, request=request):
+    if authenticated_user_from_request(request):
         return True
 
     return _public_tag_listing_enabled()

archivebox/api/v1_crawls.py

@@ -24,7 +24,7 @@ from archivebox.config.common import get_config
 from archivebox.crawls.models import Crawl
 from archivebox.misc.util import filter_queryset_by_uuid_substring
 
-from .auth import API_AUTH_METHODS, auth_using_token
+from .auth import API_AUTH_METHODS, authenticated_user_from_request
 
 router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
 
@@ -137,12 +137,16 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
     return crawl
 
 
+def get_crawl_by_ref(crawl_id: str):
+    return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+
+
 @router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
 def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
     """Get a specific Crawl by id."""
     setattr(request, "with_snapshots", with_snapshots)
     setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
 
     if crawl and as_rss:
         query = request.GET.copy()
@@ -156,33 +160,18 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn
 def crawl_file(request: HttpRequest, crawl_id: str, path: str):
     # Try to resolve the crawl first; if it doesn't exist, return 404.
     try:
-        crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+        crawl = get_crawl_by_ref(crawl_id)
     except Crawl.DoesNotExist:
         raise HttpError(404, "Crawl not found")
 
-    # Determine the effective viewer: session user takes precedence, otherwise
-    # fall back to an API token passed via ?api_key=, X-ArchiveBox-API-Key, or
-    # Authorization: Bearer ... (so that programmatic clients still work).
-    user = request.user
-    is_authenticated = bool(user.is_authenticated and user.is_active)
-    if not is_authenticated:
-        token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-        auth_header = request.headers.get("Authorization", "")
-        if not token and auth_header.lower().startswith("bearer "):
-            token = auth_header.split(None, 1)[1].strip()
-        token_user = auth_using_token(token=token, request=request) if token else None
-        if token_user and token_user.is_active:
-            user = token_user
-            is_authenticated = True
-            # Re-bind so is_admin_user() / ownership checks below see the token user.
-            setattr(request, "user", token_user)
+    user = authenticated_user_from_request(request)
 
     # Gate access using the same model as SnapshotView/can_view_snapshot:
     # admins always pass; owners can see their own crawls; otherwise the crawl
     # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
     if not is_admin_user(request):
         permissions = normalize_permissions(crawl.permissions)
-        is_owner = bool(is_authenticated and crawl.created_by_id == user.id)
+        is_owner = bool(user and crawl.created_by_id == user.id)
         if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
             raise HttpError(404, "Crawl not found")
 
@@ -217,7 +206,7 @@ def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfol
 @router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
 def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
     """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
 
@@ -259,7 +248,7 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
 
 @router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
 def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
     crawl_id_str = str(crawl.id)
     snapshot_count = crawl.snapshot_set.count()
     deleted_count, _ = crawl.delete()

archivebox/api/v1_personas.py

@@ -8,8 +8,10 @@ from uuid import UUID
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.pagination import paginate
 from pydantic import Field
 
+from archivebox.api.v1_core import CustomPagination
 from archivebox.personas.importers import validate_persona_name
 from archivebox.personas.models import Persona
 
@@ -113,6 +115,7 @@ def find_persona(extension_persona_id: str, name: str) -> Persona | None:
 
 
 @router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
+@paginate(CustomPagination)
 def get_personas(request: HttpRequest):
     """List personas available on this ArchiveBox server."""
     return Persona.objects.all().order_by("name")

archivebox/base_models/models.py

@@ -115,6 +115,10 @@ class ModelWithHealthStats(models.Model):
     class Meta(TypedModelMeta):
         abstract = True
 
+    @property
+    def admin_change_url(self) -> str:
+        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
+
     @property
     def health(self) -> int:
         total = max(self.num_uses_failed + self.num_uses_succeeded, 1)

archivebox/cli/__init__.py

@@ -66,10 +66,6 @@ class ArchiveBoxGroup(click.Group):
         # Introspection commands
         "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
     }
-    legacy_model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl_compat.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot_compat.main",
-    }
     all_subcommands = {
         **meta_commands,
         **setup_commands,
@@ -81,34 +77,29 @@ class ArchiveBoxGroup(click.Group):
         "import": "add",
         "archive": "add",
     }
-    legacy_model_subcommands = {
-        "crawl": {"create", "list", "update", "delete"},
-        "snapshot": {"create", "list", "update", "delete"},
-    }
 
     @classmethod
     def get_canonical_name(cls, cmd_name):
         return cls.renamed_commands.get(cmd_name, cmd_name)
 
     @classmethod
-    def _should_use_legacy_model_command(cls, cmd_name: str) -> bool:
-        if cmd_name not in cls.legacy_model_commands:
-            return False
+    def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool:
+        wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
+        return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands)
 
-        try:
-            arg_idx = sys.argv.index(cmd_name)
-        except ValueError:
-            return False
+    @classmethod
+    def _setup_django_for_lazy_import(cls, cmd_name: str) -> None:
+        if not cls._needs_django_for_lazy_import(cmd_name):
+            return
 
-        remaining_args = sys.argv[arg_idx + 1 :]
-        if not remaining_args:
-            return False
+        from django.apps import apps
 
-        first_arg = remaining_args[0]
-        if first_arg in ("-h", "--help"):
-            return False
+        if apps.ready:
+            return
 
-        return first_arg not in cls.legacy_model_subcommands[cmd_name]
+        from archivebox.config.django import setup_django
+
+        setup_django()
 
     def get_command(self, ctx, cmd_name):
         # handle renamed commands
@@ -120,11 +111,9 @@ class ArchiveBoxGroup(click.Group):
             cmd_name = new_name
             ctx.invoked_subcommand = cmd_name
 
-        if self._should_use_legacy_model_command(cmd_name):
-            return self._lazy_load(self.legacy_model_commands[cmd_name])
-
         # handle lazy loading of commands
         if cmd_name in self.all_subcommands:
+            self._setup_django_for_lazy_import(cmd_name)
             return self._lazy_load(cmd_name)
 
         # fall-back to using click's default command lookup
@@ -199,15 +188,12 @@ def cli(ctx, help=False):
                 raise
 
 
-def main(args=None, prog_name=None, stdin=None):
+def main(args=None, prog_name=None):
     # show `docker run archivebox xyz` in help messages if running in docker
     IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
     IS_TTY = sys.stdin.isatty()
     prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")
 
-    # stdin param allows passing input data from caller (used by __main__.py)
-    # currently not used by click-based CLI, but kept for backwards compatibility
-
     previous_unraisablehook = sys.unraisablehook
 
     def ignore_shutdown_unraisable(unraisable):

archivebox/cli/archivebox_add.py

@@ -5,7 +5,6 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox add"
 
 import sys
-import json
 import os
 from pathlib import Path
 
@@ -13,6 +12,7 @@ from typing import Any, TYPE_CHECKING
 
 import rich_click as click
 
+from archivebox.config import CONSTANTS
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.util import parse_filesize_to_bytes
 
@@ -23,21 +23,28 @@ if TYPE_CHECKING:
     from archivebox.core.models import Snapshot
 
 
-def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
+def _collect_input_urls(args: tuple[str, ...], *, parser: str = "auto") -> list[str]:
     from archivebox.misc.jsonl import read_args_or_stdin
+    from archivebox.misc.util import validate_url
 
     urls: list[str] = []
     for record in read_args_or_stdin(args):
         url = record.get("url")
         if isinstance(url, str) and url:
-            urls.append(url)
+            try:
+                urls.append(validate_url(url))
+            except ValueError as err:
+                raise click.BadParameter(str(err), param_hint="URL") from err
 
         urls_field = record.get("urls")
         if isinstance(urls_field, str):
             for line in urls_field.splitlines():
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    urls.append(line)
+                    try:
+                        urls.append(validate_url(line))
+                    except ValueError as err:
+                        raise click.BadParameter(str(err), param_hint="URL") from err
 
     return urls
 
@@ -82,7 +89,6 @@ def add(
     crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
     crawl_timeout = int(crawl_timeout or 0)
     snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
-    from archivebox import CONSTANTS
     from archivebox.config.permissions import USER, HOSTNAME
     from archivebox.config.common import get_config
 
@@ -108,6 +114,7 @@ def add(
 
     # import models once django is set up
     from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.personas.models import Persona
     from archivebox.misc.logging_util import printable_filesize
@@ -118,22 +125,7 @@ def add(
     created_by_id = created_by_id or get_or_create_system_user_pk()
     started_at = timezone.now()
 
-    if isinstance(urls, str):
-        url_list = [line.strip() for line in urls.splitlines() if line.strip()]
-    else:
-        url_list = [str(url).strip() for url in urls if str(url).strip()]
-    if snapshot_ids and len(snapshot_ids) != len(url_list):
-        raise ValueError("snapshot_ids length must match urls length")
-
-    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
-    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
-    sources_file.parent.mkdir(parents=True, exist_ok=True)
-    if snapshot_ids:
-        sources_file.write_text(
-            "\n".join(json.dumps({"url": url, "id": snapshot_ids[index], "tags": tag, "depth": 0}) for index, url in enumerate(url_list)),
-        )
-    else:
-        sources_file.write_text("\n".join(url_list))
+    source_text = urls if isinstance(urls, str) else "\n".join(str(url) for url in urls)
 
     # 2. Create a new Crawl with inline URLs
     cli_args = [*sys.argv]
@@ -143,8 +135,6 @@ def add(
 
     timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
 
-    # Read URLs directly into crawl
-    urls_content = sources_file.read_text()
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
     persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
@@ -174,9 +164,16 @@ def add(
     # stripped by Crawl.save() and rederived when hooks run.
     crawl_config.update(config_overrides)
 
+    # Crawl.urls is the user's submitted source, not a derived work queue for
+    # this add path. Keeping it byte-for-byte readable is what lets API/UI/CLI
+    # callers audit or resume imports without losing RSS/Netscape/JSON metadata
+    # that is not representable as one plain URL per line.
     crawl = Crawl.objects.create(
-        urls=urls_content,
-        max_depth=depth,
+        urls=source_text,
+        # The internal root snapshot occupies depth 0. URLs discovered from the
+        # submitted source become normal child snapshots at depth 1, so the
+        # effective crawl limit must be one hop deeper than the user requested.
+        max_depth=depth + 1,
         tags_str=tag,
         persona_id=persona_obj.id,
         label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
@@ -186,6 +183,19 @@ def add(
         config=crawl_config,
     )
 
+    # Parser plugins consume this root snapshot through the normal Snapshot
+    # hook lifecycle. ArchiveBox does not select parser plugins or call them
+    # directly; non-parser plugins cheaply no-result unsupported internal input.
+    root_snapshot = Snapshot.objects.create(
+        url=Snapshot.INTERNAL_INPUT_URL,
+        crawl=crawl,
+        depth=0,
+        title="stdin.txt",
+    )
+    staticfile_dir = root_snapshot.output_dir / "staticfile"
+    staticfile_dir.mkdir(parents=True, exist_ok=True)
+    (staticfile_dir / "stdin.txt").write_text(source_text, encoding="utf-8")
+
     print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
     first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
     print(f"    [dim]First URL: {first_url}[/dim]")
@@ -214,8 +224,8 @@ def add(
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
         from archivebox.machine.models import Process
-        from archivebox.core.takeover_util import command_owns_runtime_stack, current_command, standby_until_runtime_stack_needed
-        from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
+        from archivebox.core.takeover_util import command_owns_foreground_runner, current_command, standby_until_foreground_runner_needed
+        from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process
 
         command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
         exit_code = 0
@@ -223,15 +233,27 @@ def add(
             try:
                 with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
                     while True:
-                        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
                         exit_code = run_runner_worker(
                             ["--crawl-id", str(crawl.id)],
                             name=f"worker_runner_add_{os.getpid()}",
                             interactive_interrupts=True,
                         )
+                        crawl.refresh_from_db(fields=["status", "retry_at"])
+                        if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
+                            break
+                        # A shared supervisord can be stopped by another
+                        # foreground owner (for example `archivebox server`
+                        # shutting down) and stop the one-shot add runner while
+                        # this add's crawl is still runnable. Keep the add
+                        # process alive so it can re-enter the normal
+                        # foreground-runner ownership loop and finish its crawl.
+                        supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None
+                        if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone):
+                            continue
                         if exit_code == 0:
                             break
-                        if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+                        if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
                             continue
                         raise SystemExit(exit_code)
             except KeyboardInterrupt:
@@ -333,7 +355,12 @@ def main(**kwargs):
 
     with foreground_shutdown_signals(), foreground_parent_watchdog():
         raw_urls = kwargs.pop("urls")
-        urls = _collect_input_urls(raw_urls)
+        if raw_urls:
+            urls = _collect_input_urls(raw_urls, parser=kwargs.get("parser", "auto"))
+        elif not sys.stdin.isatty():
+            urls = sys.stdin.read()
+        else:
+            urls = []
         if not urls:
             raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
         if int(kwargs.get("max_urls") or 0) < 0:

archivebox/cli/archivebox_config.py

@@ -6,6 +6,7 @@ import sys
 import toml
 import rich_click as click
 from rich import print
+from pathlib import Path
 
 from archivebox.misc.util import docstring, enforce_types
 from archivebox.misc.toml_util import CustomTOMLEncoder
@@ -30,12 +31,17 @@ def config(
     from archivebox.misc.logging_util import printable_config
     from abx_plugins.plugins.base.utils import resolve_alias
     from archivebox.config.collection import write_config_file
+    from archivebox.config import CONSTANTS_CONFIG
     from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
     from archivebox.plugins.discovery import discover_plugin_configs
 
     check_data_folder()
 
     FLAT_CONFIG = get_config().as_dict()
+    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys()
+    readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)}
+    writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config}
+    readable_config = {**writable_config, **readonly_config}
     CONFIGS = get_all_configs()
     plugin_schemas = {
         plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
@@ -56,18 +62,23 @@ def config(
             config_options = [
                 core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
             ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
             for config_section in CONFIGS.values():
                 aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias}
 
                 for search_key in config_options:
                     # search all aliases in the section
                     for alias_key, key in aliases.items():
-                        if search_key.lower() in alias_key.lower():
+                        if key in readable_config and search_key.lower() in alias_key.lower():
                             matching_config[key] = dict(config_section)[key]
 
                     # search all keys and values in the section
                     for existing_key, value in dict(config_section).items():
+                        if existing_key in readable_config and (
+                            search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower()
+                        ):
+                            matching_config[existing_key] = value
+                    for existing_key, value in readonly_config.items():
                         if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
                             matching_config[existing_key] = value
 
@@ -79,14 +90,14 @@ def config(
             config_options = [
                 core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
             ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
+            failed_config = [key for key in config_options if key not in readable_config]
             if failed_config:
                 print("\n[red][X] These options failed to get[/red]")
                 print("    {}".format("\n    ".join(config_options)))
                 raise SystemExit(1)
         else:
-            matching_config = FLAT_CONFIG
+            matching_config = readable_config
 
         # Display core config sections
         for config_section in CONFIGS.values():
@@ -100,12 +111,18 @@ def config(
             print(_format_toml(kv_in_section))
             print("[grey53]################################################################[/grey53]")
 
+        readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config}
+        if readonly_keys:
+            print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
+            print(_format_toml(readonly_keys))
+            print("[grey53]################################################################[/grey53]")
+
         plugin_keys = {}
 
         # Collect all plugin config keys
         for schema in plugin_schemas.values():
             for key in schema.keys():
-                if key in matching_config:
+                if key in matching_config and key in writable_config:
                     plugin_keys[key] = matching_config[key]
 
         # Display all plugin config in single [PLUGINS] section
@@ -135,7 +152,7 @@ def config(
                     f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
                 )
 
-            if key in FLAT_CONFIG:
+            if key in writable_config:
                 new_config[key] = val.strip()
             else:
                 failed_options.append(line)

archivebox/cli/archivebox_crawl.py

@@ -65,6 +65,7 @@ def create_crawl(
         1: Failure
     """
     from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
+    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
 
@@ -98,7 +99,11 @@ def create_crawl(
         # Collect URLs
         url = record.get("url")
         if url:
-            url_list.append(url)
+            try:
+                url_list.append(validate_url(str(url)))
+            except ValueError as err:
+                rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                return 1
 
         # Handle 'urls' field (newline-separated)
         urls_field = record.get("urls")
@@ -106,7 +111,11 @@ def create_crawl(
             for line in urls_field.split("\n"):
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    url_list.append(line)
+                    try:
+                        url_list.append(validate_url(line))
+                    except ValueError as err:
+                        rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                        return 1
 
     # Output pass-through records first
     if not is_tty:

archivebox/cli/archivebox_crawl_compat.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_add import add
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--wait/--no-wait", "wait", default=True, help="Accepted for backwards compatibility")
-@click.argument("urls", nargs=-1)
-def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox crawl URL...` entrypoint."""
-    del status, wait
-    add(list(urls), depth=depth, tag=tag, bg=True)
-    sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_extract.py

@@ -48,10 +48,11 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     from rich import print as rprint
     from django.utils import timezone
     from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.services.runner import run_crawl
 
     try:
-        archiveresult = ArchiveResult.objects.get(id=archiveresult_id)
+        archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
     except ArchiveResult.DoesNotExist:
         rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
         return 1
@@ -210,7 +211,7 @@ def run_plugins(
             if snapshot_id in existing_snapshot_ids
             for plugin_name in plugin_names
         )
-    plugins_by_name = discover_plugins()
+    plugins_by_name = discover_plugins(runtime="archivebox")
     requested_rows: set[tuple[str, str, str]] = set()
     for snapshot_id, plugin_name in requested_pairs:
         plugin = plugins_by_name.get(plugin_name)
@@ -418,16 +419,11 @@ def run_plugins(
 
 
 def is_archiveresult_id(value: str) -> bool:
-    """Check if value looks like an ArchiveResult UUID."""
-    import re
-
-    uuid_pattern = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
-    if not uuid_pattern.match(value):
-        return False
-    # Verify it's actually an ArchiveResult (not a Snapshot or other object)
+    """Check if value resolves to an ArchiveResult ID."""
     from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
 
-    return ArchiveResult.objects.filter(id=value).exists()
+    return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists()
 
 
 @click.command()

archivebox/cli/archivebox_help.py

@@ -13,6 +13,10 @@ from rich.panel import Panel
 
 
 def _command_doc(cmd: str, import_path: str) -> str:
+    def first_doc_line(docstring: str | None) -> str:
+        lines = (docstring or "").splitlines()
+        return lines[0] if lines else ""
+
     modname, _ = import_path.rsplit(".", 1)
     spec = importlib.util.find_spec(modname)
     if spec is None or spec.origin is None:
@@ -26,9 +30,9 @@ def _command_doc(cmd: str, import_path: str) -> str:
     for name in (cmd, "main"):
         for node in tree.body:
             if isinstance(node, ast.FunctionDef) and node.name == name:
-                return (ast.get_docstring(node) or "").splitlines()[0]
+                return first_doc_line(ast.get_docstring(node))
 
-    return (ast.get_docstring(tree) or "").splitlines()[0]
+    return first_doc_line(ast.get_docstring(tree))
 
 
 def help() -> None:

archivebox/cli/archivebox_init.py

@@ -94,10 +94,18 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     from archivebox.config.django import setup_django
 
     setup_django()
-    check_migrations(blocking=True, auto_apply=False)
+    previous_wants_init = os.environ.get("ARCHIVEBOX_WANTS_INIT")
+    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+    try:
+        check_migrations(blocking=True, auto_apply=False)
 
-    for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
-        sys.stdout.write(f"    {migration_line}\n")
+        for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
+            sys.stdout.write(f"    {migration_line}\n")
+    finally:
+        if previous_wants_init is None:
+            os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
+        else:
+            os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
 
     assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
     print()
@@ -155,7 +163,6 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
     if working_lib_dir:
         working_lib_dir.mkdir(parents=True, exist_ok=True)
-        (working_lib_dir / "bin").mkdir(parents=True, exist_ok=True)
 
     if install:
         from archivebox.cli.archivebox_install import install as install_method

archivebox/cli/archivebox_list.py

@@ -7,50 +7,15 @@ import sys
 
 import rich_click as click
 
-from archivebox.cli.archivebox_snapshot import list_snapshots
+from archivebox.cli.archivebox_snapshot import list_snapshots, snapshot_filter_options, snapshot_output_options
 
 
 @click.command()
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def main(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-) -> None:
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def main(**kwargs) -> None:
     """List Snapshots."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_remove.py

@@ -23,6 +23,7 @@ from archivebox.misc.logging_util import (
     log_removal_finished,
     TimedProgress,
 )
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
 
 @enforce_types
@@ -34,48 +35,54 @@ def remove(
     before: float | None = None,
     yes: bool = False,
     out_dir: Path = CONSTANTS.DATA_DIR,
-) -> QuerySet:
+    timeout: float | None = None,
+    **kwargs,
+) -> dict[str, object]:
     """Remove the specified URLs from the archive"""
 
     setup_django()
     check_data_folder()
+    timeout = float(timeout) if timeout is not None else None
 
-    from archivebox.cli.archivebox_search import get_snapshots
+    from archivebox.core.models import Snapshot
 
+    filter_kwargs = {
+        **kwargs,
+        "filter_patterns": filter_patterns,
+        "filter_type": filter_type,
+        "after": after,
+        "before": before,
+    }
     pattern_list = list(filter_patterns)
 
     log_list_started(pattern_list or None, filter_type)
     timer = TimedProgress(360, prefix="      ")
     try:
-        snapshots = get_snapshots(
-            snapshots=snapshots,
-            filter_patterns=pattern_list or None,
-            filter_type=filter_type,
-            after=after,
-            before=before,
-        )
+        if snapshots is None:
+            snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
+        # Freeze the target set up-front so a concurrent daemon writing new
+        # snapshots can't extend the deletion under us, and so the cursor isn't
+        # held open across the per-row deletes below.
+        snapshot_pks = list(snapshots.values_list("pk", flat=True))
     finally:
         timer.end()
 
-    if not snapshots.exists():
+    if not snapshot_pks:
         log_removal_finished(0, 0)
         raise SystemExit(1)
 
-    log_list_finished(snapshots)
-    log_removal_started(snapshots, yes=yes)
+    if not yes:
+        log_list_finished(snapshots)
+        log_removal_started(snapshots, yes=False)
 
-    from archivebox.core.models import Snapshot
     from archivebox.search.query import flush_search_index
 
-    # Freeze the target set up-front so a concurrent daemon writing new
-    # snapshots can't extend the deletion under us, and so the cursor isn't
-    # held open across the per-row deletes below.
-    snapshot_pks = list(snapshots.values_list("pk", flat=True))
-    to_remove = len(snapshot_pks)
+    started_at = time.monotonic()
+    deadline = started_at + timeout if timeout is not None else None
 
     # Search-index flush touches a separate backend (FTS / sonic), not the
     # main index.sqlite3 writer lock, so it's safe to do once up front.
-    flush_search_index(snapshots=Snapshot.objects.filter(pk__in=snapshot_pks))
+    flush_search_index(snapshots=snapshots)
 
     # Delete one snapshot at a time. Each ``.delete()`` is its own short
     # Django-atomic block, so the writer lock is released between rows and
@@ -90,40 +97,59 @@ def remove(
     # own retry loop at this outer (non-atomic) level. Each attempt is a
     # fresh atomic; an exception cleanly rolls it back before we sleep.
     retry_interval = 1.0
-    retry_timeout = 60.0
-    for pk in snapshot_pks:
-        deadline = time.monotonic() + retry_timeout
+    deleted_snapshot_pks = []
+    timed_out = False
+    timeout_error = ""
+    for index, pk in enumerate(snapshot_pks):
+        if deadline is not None and time.monotonic() >= deadline:
+            timed_out = True
+            timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining."
+            break
         while True:
             try:
-                Snapshot.objects.filter(pk=pk).delete()
+                deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
+                if deleted_count:
+                    deleted_snapshot_pks.append(pk)
                 break
             except OperationalError as err:
-                if "database is locked" not in str(err) or time.monotonic() >= deadline:
+                if "database is locked" not in str(err):
                     raise
-                time.sleep(retry_interval)
+                remaining_time = deadline - time.monotonic() if deadline is not None else None
+                if remaining_time is not None and remaining_time <= 0:
+                    timed_out = True
+                    timeout_error = f"Remove timed out after {timeout:g}s while waiting for the database lock."
+                    break
+                time.sleep(min(retry_interval, remaining_time) if remaining_time is not None else retry_interval)
+        if timed_out:
+            break
 
     all_snapshots = Snapshot.objects.all()
-    log_removal_finished(all_snapshots.count(), to_remove)
+    remaining_count = all_snapshots.count()
+    deleted_snapshot_id_set = set(deleted_snapshot_pks)
+    remaining_snapshot_pks = [snapshot_id for snapshot_id in snapshot_pks if snapshot_id not in deleted_snapshot_id_set]
+    log_removal_finished(remaining_count, len(deleted_snapshot_pks))
 
-    return all_snapshots
+    return {
+        "removed_count": len(deleted_snapshot_pks),
+        "removed_snapshot_ids": [str(snapshot_id) for snapshot_id in deleted_snapshot_pks],
+        "not_removed_count": len(remaining_snapshot_pks),
+        "not_removed_snapshot_ids": [str(snapshot_id) for snapshot_id in remaining_snapshot_pks],
+        "success": not timed_out,
+        "error": timeout_error,
+        "timeout": timeout,
+    }
 
 
 @click.command()
 @click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
-@click.option("--before", type=float, help="Remove only URLs bookmarked before timestamp")
-@click.option("--after", type=float, help="Remove only URLs bookmarked after timestamp")
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(("exact", "substring", "domain", "regex", "tag")),
-    default="exact",
-    help="Type of pattern matching to use when filtering URLs",
-)
-@click.argument("filter_patterns", nargs=-1)
+@click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots")
+@snapshot_filter_options(default_filter_type="exact")
 @docstring(remove.__doc__)
 def main(**kwargs):
     """Remove the specified URLs from the archive"""
-    remove(**kwargs)
+    result = remove(**kwargs)
+    if not result["success"]:
+        raise SystemExit(124)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_run.py

@@ -243,11 +243,17 @@ def process_stdin_records() -> int:
                     crawl_id,
                     snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
                     selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
+                    selected_plugins_are_explicit=False,
                 )
     return 0
 
 
-def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int:
+def run_runner(
+    daemon: bool = False,
+    crawl_id: str | None = None,
+    maintenance_only: bool = False,
+    maintenance_batch_size: int | None = None,
+) -> int:
     """
     Run the background runner loop.
 
@@ -259,11 +265,24 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
     from archivebox.config import CONSTANTS
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Machine, Process
-    from archivebox.core.takeover_util import enter_single_runner_gate
-    from archivebox.services.runner import recover_orchestrator_state, run_pending_crawls
+    from archivebox.core.takeover_util import enter_single_runner_gate, standby_until_foreground_runner_needed
+    from archivebox.core.recovery_util import recover_orchestrator_state
+    from archivebox.services.runner import run_pending_crawls
 
     Machine.current()
     current = Process.current()
+    root_command = current.root
+    if daemon and root_command.process_type in (
+        Process.TypeChoices.SERVER,
+        Process.TypeChoices.ADD,
+        Process.TypeChoices.UPDATE,
+    ):
+        # Server-owned daemon runners are persistent supervisor workers, but
+        # foreground add/update commands are allowed to borrow runner/sonic
+        # leadership without taking down Daphne. Waiting here keeps the worker
+        # on the normal runner path while preventing a server restart loop from
+        # immediately stealing the single-runner gate back from the newer CLI.
+        standby_until_foreground_runner_needed(root_command, data_dir=CONSTANTS.DATA_DIR)
     if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
         current.mark_exited()
         return 0
@@ -305,9 +324,15 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
                 crawl_id=crawl_id,
                 maintenance_only=maintenance_only,
                 interactive_interrupts=interactive_interrupts,
+                **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
             )
         return 0
-    except (KeyboardInterrupt, asyncio.CancelledError):
+    except KeyboardInterrupt:
+        return 0
+    except asyncio.CancelledError as e:
+        if daemon:
+            rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+            return 1
         return 0
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
@@ -324,7 +349,23 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
 @click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
-def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool):
+@click.option(
+    "--maintenance-batch-size",
+    type=int,
+    default=None,
+    hidden=True,
+    help="Limit queued maintenance snapshots claimed per scheduler tick",
+)
+@click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
+def main(
+    daemon: bool,
+    crawl_id: str,
+    snapshot_id: str,
+    binary_id: str,
+    maintenance_only: bool,
+    maintenance_batch_size: int | None,
+    no_stdin: bool,
+):
     """
     Process queued work.
 
@@ -347,7 +388,7 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
                 ),
                 foreground_parent_watchdog(enabled=False),
             ):
-                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
+                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
         except KeyboardInterrupt:
             sys.exit(0)
 
@@ -371,30 +412,49 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
                 sys.exit(1)
 
         if crawl_id:
-            sys.exit(run_runner(daemon=False, crawl_id=crawl_id, maintenance_only=maintenance_only))
+            sys.exit(
+                run_runner(
+                    daemon=False,
+                    crawl_id=crawl_id,
+                    maintenance_only=maintenance_only,
+                    maintenance_batch_size=maintenance_batch_size,
+                ),
+            )
 
         if maintenance_only:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=True))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=True, maintenance_batch_size=maintenance_batch_size))
 
-        if not sys.stdin.isatty():
+        if not no_stdin and not sys.stdin.isatty():
             sys.exit(process_stdin_records())
         else:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
 
 
 def run_snapshot_worker(snapshot_id: str) -> int:
+    from archivebox.config import CONSTANTS
+    from archivebox.core.takeover_util import enter_single_runner_gate
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    from archivebox.machine.models import Process
     from archivebox.core.models import Snapshot
     from archivebox.services.runner import run_due_snapshot
     from django.utils import timezone
 
+    current = Process.current()
+    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
+        current.mark_exited()
+        return 0
+
     snapshot = None
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
-            snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-            if snapshot.retry_at is None:
-                snapshot.update_and_requeue(retry_at=timezone.now())
-            run_due_snapshot(snapshot, lock_seconds=60)
+            for _ in range(10):
+                snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+                if snapshot.retry_at is None:
+                    snapshot.update_and_requeue(retry_at=timezone.now())
+                elif snapshot.retry_at > timezone.now():
+                    break
+                if not run_due_snapshot(snapshot, lock_seconds=60):
+                    break
         return 0
     except KeyboardInterrupt:
         try:
@@ -413,6 +473,10 @@ def run_snapshot_worker(snapshot_id: str) -> int:
 
         traceback.print_exc()
         return 1
+    finally:
+        current.refresh_from_db()
+        if current.status != Process.StatusChoices.EXITED:
+            current.mark_exited()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_search.py

@@ -3,255 +3,7 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox search"
 
-import sys
-from pathlib import Path
-from typing import TYPE_CHECKING
-from collections.abc import Callable
-
-import rich_click as click
-
-from django.db.models import Q, QuerySet
-
-from archivebox.config import CONSTANTS
-from archivebox.config.common import get_config
-from archivebox.misc.logging import stderr
-from archivebox.misc.util import enforce_types, docstring
-
-if TYPE_CHECKING:
-    from archivebox.core.models import Snapshot
-
-# Filter types for URL matching
-LINK_FILTERS: dict[str, Callable[[str], Q]] = {
-    "exact": lambda pattern: Q(url=pattern),
-    "substring": lambda pattern: Q(url__icontains=pattern),
-    "regex": lambda pattern: Q(url__iregex=pattern),
-    "domain": lambda pattern: (
-        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
-    ),
-    "tag": lambda pattern: Q(tags__name=pattern),
-    "timestamp": lambda pattern: Q(timestamp=pattern),
-}
-
-STATUS_CHOICES = ["indexed", "archived", "unarchived"]
-
-
-def _apply_pattern_filters(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    filter_patterns: list[str],
-    filter_type: str,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    filter_builder = LINK_FILTERS.get(filter_type)
-    if filter_builder is None:
-        stderr()
-        stderr(f"[X] Got invalid pattern for --filter-type={filter_type}", color="red")
-        raise SystemExit(2)
-
-    query = Q()
-    for pattern in filter_patterns:
-        query |= filter_builder(pattern)
-    return snapshots.filter(query)
-
-
-def _snapshots_to_json(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from archivebox.config import VERSION
-    from archivebox.misc.util import to_json
-
-    config = get_config()
-    main_index_header = (
-        {
-            "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
-            "schema": "archivebox.index.json",
-            "copyright_info": config.FOOTER_INFO,
-            "meta": {
-                "project": "ArchiveBox",
-                "version": VERSION,
-                "git_sha": VERSION,
-                "website": "https://ArchiveBox.io",
-                "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
-                "source": "https://github.com/ArchiveBox/ArchiveBox",
-                "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
-                "dependencies": {},
-            },
-        }
-        if with_headers
-        else {}
-    )
-
-    snapshot_dicts = [snapshot.to_dict(extended=True) for snapshot in snapshots.iterator(chunk_size=500)]
-    output: dict[str, object] | list[dict[str, object]]
-    if with_headers:
-        output = {
-            **main_index_header,
-            "num_links": len(snapshot_dicts),
-            "updated": datetime.now(tz.utc),
-            "last_run_cmd": sys.argv,
-            "links": snapshot_dicts,
-        }
-    else:
-        output = snapshot_dicts
-
-    return to_json(output, indent=4, sort_keys=True)
-
-
-def _snapshots_to_csv(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    cols: list[str],
-    with_headers: bool,
-) -> str:
-    header = ",".join(cols) if with_headers else ""
-    rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.iterator(chunk_size=500)]
-    return "\n".join((header, *rows))
-
-
-def _snapshots_to_html(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from django.template.loader import render_to_string
-
-    from archivebox.config import VERSION
-    from archivebox.config.version import get_COMMIT_HASH
-
-    config = get_config()
-    template = "static_index.html" if with_headers else "minimal_index.html"
-    snapshot_list = list(snapshots.iterator(chunk_size=500))
-
-    return render_to_string(
-        template,
-        {
-            "version": VERSION,
-            "git_sha": get_COMMIT_HASH() or VERSION,
-            "num_links": str(len(snapshot_list)),
-            "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"),
-            "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"),
-            "links": snapshot_list,
-            "FOOTER_INFO": config.FOOTER_INFO,
-        },
-    )
-
-
-def get_snapshots(
-    snapshots: QuerySet["Snapshot", "Snapshot"] | None = None,
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    after: float | None = None,
-    before: float | None = None,
-    out_dir: Path = CONSTANTS.DATA_DIR,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    """Filter and return Snapshots matching the given criteria."""
-    from archivebox.core.models import Snapshot
-
-    if snapshots is not None:
-        result = snapshots
-    else:
-        result = Snapshot.objects.all()
-
-    if after is not None:
-        result = result.filter(timestamp__gte=after)
-    if before is not None:
-        result = result.filter(timestamp__lt=before)
-    if filter_patterns:
-        result = _apply_pattern_filters(result, filter_patterns, filter_type)
-
-    # Prefetch crawl relationship to avoid N+1 queries when accessing output_dir
-    result = result.select_related("crawl", "crawl__created_by")
-
-    if not result.exists():
-        stderr("[!] No Snapshots matched your filters:", filter_patterns, f"({filter_type})", color="lightyellow")
-
-    return result
-
-
-@enforce_types
-def search(
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    status: str = "indexed",
-    before: float | None = None,
-    after: float | None = None,
-    sort: str | None = None,
-    json: bool = False,
-    html: bool = False,
-    csv: str | None = None,
-    with_headers: bool = False,
-):
-    """List, filter, and export information about archive entries"""
-
-    if with_headers and not (json or html or csv):
-        stderr("[X] --with-headers requires --json, --html or --csv\n", color="red")
-        raise SystemExit(2)
-
-    # Query DB directly - no filesystem scanning
-    snapshots = get_snapshots(
-        filter_patterns=list(filter_patterns) if filter_patterns else None,
-        filter_type=filter_type,
-        before=before,
-        after=after,
-    )
-
-    # Apply status filter
-    if status == "archived":
-        snapshots = snapshots.filter(downloaded_at__isnull=False)
-    elif status == "unarchived":
-        snapshots = snapshots.filter(downloaded_at__isnull=True)
-    # 'indexed' = all snapshots (no filter)
-
-    if sort:
-        snapshots = snapshots.order_by(sort)
-
-    # Export to requested format
-    if json:
-        output = _snapshots_to_json(snapshots, with_headers=with_headers)
-    elif html:
-        output = _snapshots_to_html(snapshots, with_headers=with_headers)
-    elif csv:
-        output = _snapshots_to_csv(snapshots, cols=csv.split(","), with_headers=with_headers)
-    else:
-        from archivebox.misc.logging_util import printable_folders
-
-        # Convert to dict for printable_folders
-        folders: dict[str, Snapshot | None] = {str(snapshot.output_dir): snapshot for snapshot in snapshots}
-        output = printable_folders(folders, with_headers)
-
-    # Structured exports must be written directly to stdout.
-    # rich.print() reflows long lines to console width, which corrupts JSON/CSV/HTML output.
-    sys.stdout.write(output)
-    if not output.endswith("\n"):
-        sys.stdout.write("\n")
-    return output
-
-
-@click.command()
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(["search", *LINK_FILTERS.keys()]),
-    default="substring",
-    help="Pattern matching type for filtering URLs",
-)
-@click.option("--status", "-s", type=click.Choice(STATUS_CHOICES), default="indexed", help="List snapshots with the given status")
-@click.option("--before", "-b", type=float, help="List snapshots bookmarked before the given UNIX timestamp")
-@click.option("--after", "-a", type=float, help="List snapshots bookmarked after the given UNIX timestamp")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--json", "-J", is_flag=True, help="Print output in JSON format")
-@click.option("--html", "-M", is_flag=True, help="Print output in HTML format (suitable for viewing statically without a server)")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: created_at,url,title")
-@click.option("--with-headers", "-H", is_flag=True, help="Include extra CSV/HTML headers in the output")
-@click.help_option("--help", "-h")
-@click.argument("filter_patterns", nargs=-1)
-@docstring(search.__doc__)
-def main(**kwargs):
-    return search(**kwargs)
+from archivebox.cli.archivebox_list import main
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_server.py

@@ -313,6 +313,7 @@ def server(
     from archivebox.core.takeover_util import (
         command_owns_runtime_stack,
         current_command,
+        foreground_runner_owner,
         runtime_stack_owner,
         standby_until_runtime_stack_needed,
     )
@@ -354,7 +355,10 @@ def server(
         ):
             while True:
                 standby_result = standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
-                older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id)
+                older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id) or foreground_runner_owner(
+                    data_dir=CONSTANTS.DATA_DIR,
+                    exclude_id=command.id,
+                )
                 takeover_components = active_supervisord_runtime_components(config=config)
                 if older_owner and takeover_components:
                     print(

archivebox/cli/archivebox_shell.py

@@ -12,9 +12,10 @@ from archivebox.misc.util import docstring
 def shell(args: Iterable[str] = ()) -> None:
     """Enter an interactive ArchiveBox Django shell"""
 
-    from django.core.management import call_command
+    from django.core.management import call_command, get_commands
 
-    call_command("shell_plus", *args)
+    shell_command = "shell_plus" if "shell_plus" in get_commands() else "shell"
+    call_command(shell_command, *args)
 
 
 @click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))

archivebox/cli/archivebox_snapshot.py

@@ -35,9 +35,10 @@ from collections.abc import Iterable
 
 import rich_click as click
 from rich import print as rprint
-from django.db.models import Case, IntegerField, QuerySet, When
+from django.db.models import QuerySet
 
-from archivebox.cli.cli_util import apply_filters
+SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp")
+SNAPSHOT_LIST_CHUNK_SIZE = 100
 
 
 # =============================================================================
@@ -66,7 +67,7 @@ def create_snapshots(
         TYPE_SNAPSHOT,
         TYPE_CRAWL,
     )
-    from archivebox.misc.util import validate_url_length
+    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot
     from archivebox.crawls.models import Crawl
@@ -111,7 +112,7 @@ def create_snapshots(
                 # Create snapshots for each URL in the crawl
                 for url in crawl.get_urls_list():
                     try:
-                        validate_url_length(url)
+                        validate_url(url)
                     except ValueError as err:
                         rprint(f"[red]Error creating snapshot: {err}[/red]", file=sys.stderr)
                         continue
@@ -134,7 +135,7 @@ def create_snapshots(
             elif record_type == TYPE_SNAPSHOT or record.get("url"):
                 # Input is a Snapshot or plain URL
                 if record.get("url"):
-                    validate_url_length(str(record["url"]))
+                    validate_url(str(record["url"]))
                 if tag and not record.get("tags"):
                     record["tags"] = tag
                 if status:
@@ -178,72 +179,63 @@ def create_snapshots(
 # =============================================================================
 
 
+def snapshot_filter_options(*, default_filter_type: str):
+    def decorate(func):
+        for decorator in reversed(
+            (
+                click.option("--status", "-s", help="Filter by status (queued, started, sealed)"),
+                click.option("--url__icontains", help="Filter by URL contains"),
+                click.option("--url__istartswith", help="Filter by URL starts with"),
+                click.option("--tag", "-t", help="Filter by tag name"),
+                click.option("--crawl-id", help="Filter by crawl ID"),
+                click.option("--limit", "-n", type=int, help="Limit number of results"),
+                click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at"),
+                click.option("--search", help="Search mode to use for positional query"),
+                click.option("--before", type=float, help="Only snapshots bookmarked before timestamp"),
+                click.option("--after", type=float, help="Only snapshots bookmarked after timestamp"),
+                click.option(
+                    "--filter-type",
+                    "-f",
+                    type=click.Choice(SNAPSHOT_FILTER_TYPE_CHOICES),
+                    default=default_filter_type,
+                    help="Type of pattern matching to use for positional filters",
+                ),
+                click.argument("filter_patterns", nargs=-1),
+            ),
+        ):
+            func = decorator(func)
+        return func
+
+    return decorate
+
+
+def snapshot_output_options(func):
+    for decorator in reversed(
+        (
+            click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title"),
+            click.option("--json", "as_json", is_flag=True, help="Print output as a JSON array"),
+            click.option("--html", "as_html", is_flag=True, help="Print output as HTML"),
+            click.option("--with-headers", is_flag=True, help="Include column headers in structured output"),
+        ),
+    ):
+        func = decorator(func)
+    return func
+
+
 def build_snapshot_queryset(
-    *,
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    sort: str | None = None,
-    search: str | None = None,
-    query: str | None = None,
-    limit: int | None = None,
+    **kwargs,
 ) -> QuerySet:
     from archivebox.core.models import Snapshot
-    from archivebox.search.query import apply_snapshot_search
 
-    queryset = Snapshot.objects.order_by("-created_at")
-    queryset = apply_filters(
-        queryset,
-        {
-            "status": status,
-            "url__icontains": url__icontains,
-            "url__istartswith": url__istartswith,
-            "crawl_id": crawl_id,
-        },
-    )
-
-    if tag:
-        queryset = queryset.filter(tags__name__iexact=tag)
-
-    query = (query or "").strip()
-    if query:
-        try:
-            queryset = apply_snapshot_search(
-                queryset,
-                query,
-                search_mode=search,
-                ordering=("-created_at",) if not sort else None,
-                max_results=limit,
-                skip_backend_when_metadata_satisfies_limit=True,
-                include_metadata_for_forced_backend=True,
-            )
-        except Exception as err:
-            rprint(
-                f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
-                file=sys.stderr,
-            )
-            queryset = apply_snapshot_search(queryset, query, search_mode="meta")
-
-    if sort:
-        queryset = queryset.order_by(sort)
-
-    return queryset
+    return Snapshot.objects.order_by("-created_at").search(**kwargs)
 
 
 def list_snapshots(
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
     csv: str | None = None,
+    as_json: bool = False,
+    as_html: bool = False,
     with_headers: bool = False,
-    search: str | None = None,
-    query: str | None = None,
+    **kwargs,
 ) -> int:
     """
     List Snapshots as JSONL with optional filters.
@@ -252,90 +244,71 @@ def list_snapshots(
         0: Success (even if no results)
     """
     from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import Snapshot
 
-    if with_headers and not csv:
-        rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr)
+    output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
+    if output_formats > 1:
+        rprint("[red]Choose only one output format: --csv, --json, or --html[/red]", file=sys.stderr)
+        return 2
+    if with_headers and not output_formats:
+        rprint("[red]--with-headers requires --csv, --json, or --html[/red]", file=sys.stderr)
         return 2
 
-    is_tty = sys.stdout.isatty() and not csv
+    is_tty = sys.stdout.isatty() and not output_formats
 
-    queryset = build_snapshot_queryset(
-        status=status,
-        url__icontains=url__icontains,
-        url__istartswith=url__istartswith,
-        tag=tag,
-        crawl_id=crawl_id,
-        sort=sort,
-        search=search,
-        query=query,
-        limit=limit,
-    )
-
-    if not is_tty:
-        if limit:
-            limited_ids = list(queryset.values_list("id", flat=True)[:limit])
-            preserved_order = Case(
-                *(When(id=snapshot_id, then=position) for position, snapshot_id in enumerate(limited_ids)),
-                output_field=IntegerField(),
-            )
-            queryset = Snapshot.objects.filter(id__in=limited_ids).order_by(preserved_order)
-        queryset = queryset.prefetch_related("tags")
-    elif limit:
-        queryset = queryset[:limit]
+    try:
+        queryset = build_snapshot_queryset(**kwargs)
+    except ValueError as err:
+        rprint(f"[red]{err}[/red]", file=sys.stderr)
+        return 2
 
     count = 0
+    if as_json:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_json(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    if as_html:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_html(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
     if csv:
         cols = [col.strip() for col in csv.split(",") if col.strip()]
         if not cols:
             rprint("[red]No CSV columns provided[/red]", file=sys.stderr)
             return 2
-        rows: list[str] = []
         if with_headers:
-            rows.append(",".join(cols))
-        simple_cols = {
-            "id",
-            "crawl_id",
-            "url",
-            "title",
-            "timestamp",
-            "depth",
-            "status",
-            "fs_version",
-            "bookmarked_at",
-            "created_at",
-            "modified_at",
-            "retry_at",
-            "downloaded_at",
-        }
-        from archivebox.misc.util import to_json
-
-        for snapshot in queryset.iterator(chunk_size=500):
-            if set(cols).issubset(simple_cols):
-                rows.append(
-                    ",".join(to_json(snapshot.serializable_value(col), indent=None) for col in cols),
-                )
-            else:
-                rows.append(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write(",".join(cols))
+            sys.stdout.write("\n")
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+            sys.stdout.write(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write("\n")
             count += 1
-        output = "\n".join(rows)
-        if output:
-            sys.stdout.write(output)
-            if not output.endswith("\n"):
-                sys.stdout.write("\n")
         rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
         return 0
 
-    for snapshot in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(snapshot.status, "dim")
-            rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
-        else:
+    if not is_tty:
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
             write_record(snapshot.to_json())
+            count += 1
+        rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    for snapshot in queryset.iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+        status_color = {
+            "queued": "yellow",
+            "started": "blue",
+            "sealed": "green",
+        }.get(snapshot.status, "dim")
+        rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
         count += 1
 
     rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
@@ -494,46 +467,11 @@ def create_cmd(urls: tuple, tag: str, status: str, depth: int):
 
 
 @main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def list_cmd(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-):
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def list_cmd(**kwargs):
     """List Snapshots as JSONL."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))
 
 
 @main.command("update")

archivebox/cli/archivebox_snapshot_compat.py

@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox snapshot"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_snapshot import create_snapshots
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--depth", "-d", type=int, default=0, help="Crawl depth (default: 0)")
-@click.argument("urls", nargs=-1)
-def main(tag: str, status: str, depth: int, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox snapshot URL...` entrypoint."""
-    sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth))
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_status.py

@@ -105,7 +105,7 @@ def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
 
     if num_indexed:
         print("    [violet]Hint:[/violet] You can list snapshots by status like so:")
-        print("        [green]archivebox list --status=<status>  (e.g. archived, queued, etc.)[/green]")
+        print("        [green]archivebox list --status=<status>  (e.g. sealed, queued, etc.)[/green]")
 
     if orphaned_dirs:
         print("    [violet]Hint:[/violet] To automatically import orphaned data directories into the main index, run:")

archivebox/cli/archivebox_update.py

@@ -15,6 +15,7 @@ from pathlib import Path
 import rich_click as click
 
 from archivebox.misc.util import enforce_types, docstring
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
@@ -32,66 +33,36 @@ def _get_snapshot_crawl(snapshot: Snapshot) -> Crawl | None:
 
 
 def _get_search_indexing_plugins() -> list[str]:
-    from abx_dl.models import discover_plugins
+    from archivebox.config.common import get_config
+    from archivebox.plugins.hooks import discover_hooks
     from archivebox.plugins.discovery import get_search_backends
 
     available_backends = set(get_search_backends())
-    plugins = discover_plugins()
     return sorted(
         plugin_name
-        for plugin_name, plugin in plugins.items()
-        if plugin_name.startswith("search_backend_")
-        and plugin_name.removeprefix("search_backend_") in available_backends
-        and any("Snapshot" in hook.name and "index" in hook.name.lower() for hook in plugin.hooks)
+        for plugin_name in {
+            hook.parent.name
+            for hook in discover_hooks("Snapshot", config=get_config())
+            if hook.parent.name.startswith("search_backend_") and "index" in hook.name.lower()
+        }
+        if plugin_name.startswith("search_backend_") and plugin_name.removeprefix("search_backend_") in available_backends
     )
 
 
 def _build_filtered_snapshots_queryset(
-    *,
-    filter_patterns: Iterable[str],
-    filter_type: str,
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
-    search: str | None = None,
-    before: float | None = None,
-    after: float | None = None,
-    resume: str | None = None,
+    **kwargs,
 ):
-    from datetime import datetime
+    from archivebox.core.models import Snapshot
     from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
-    filter_patterns = tuple(filter_patterns)
-    snapshots = build_snapshot_queryset(
-        status=status,
-        url__icontains=url__icontains,
-        url__istartswith=url__istartswith,
-        tag=tag,
-        crawl_id=crawl_id,
-        sort=sort,
-        search=search,
-        query=" ".join(filter_patterns) if search else None,
-    )
-
-    if filter_patterns and not search:
-        snapshots = snapshots.filter_by_patterns(list(filter_patterns), filter_type)
-
-    if before:
-        snapshots = snapshots.filter(bookmarked_at__lt=datetime.fromtimestamp(before))
-    if after:
-        snapshots = snapshots.filter(bookmarked_at__gt=datetime.fromtimestamp(after))
-    if resume:
-        snapshots = snapshots.filter(timestamp__lte=resume)
-    if not sort:
-        snapshots = snapshots.order_by("-timestamp")
+    limit = kwargs.pop("limit", None)
+    snapshots = build_snapshot_queryset(**kwargs)
+    if kwargs.get("resume"):
+        snapshots = snapshots.filter(timestamp__lte=kwargs["resume"])
     snapshots = snapshots.select_related("crawl")
-    if limit:
-        limited_ids = list(snapshots.values_list("id", flat=True)[:limit])
-        snapshots = snapshots.model.objects.filter(id__in=limited_ids).select_related("crawl")
+    if limit is not None and limit > 0:
+        snapshot_ids = list(snapshots.values_list("id", flat=True)[:limit])
+        snapshots = Snapshot.objects.filter(id__in=snapshot_ids).select_related("crawl")
 
     return snapshots
 
@@ -110,7 +81,7 @@ def reindex_snapshots(
 
     stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
     records: list[dict[str, str]] = []
-    plugins_by_name = discover_plugins()
+    plugins_by_name = discover_plugins(runtime="archivebox")
     required_hooks_by_plugin = {
         plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
         for plugin_name in search_plugins
@@ -254,27 +225,42 @@ def update(
     from archivebox.machine.models import Process
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals, raise_if_shutdown_requested
     from archivebox.core.takeover_util import (
-        command_owns_runtime_stack,
+        command_owns_foreground_runner,
         current_command,
         ensure_daemon_stack,
-        standby_until_runtime_stack_needed,
+        foreground_runner_owner,
+        standby_until_foreground_runner_needed,
     )
-    from archivebox.workers.supervisord_util import run_runner_worker, stop_existing_supervisord_process, stop_own_supervisord_process
+    from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
 
     command = current_command(Process.TypeChoices.UPDATE, data_dir=CONSTANTS.DATA_DIR)
 
+    def still_owns_foreground_runner() -> bool:
+        from django.db import connections
+
+        try:
+            return command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR)
+        finally:
+            connections.close_all()
+
     def wait_for_turn() -> None:
         raise_if_shutdown_requested()
-        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
         raise_if_shutdown_requested()
 
-    def run_scoped_runner(*args: str) -> None:
+    def run_scoped_runner(*args: str, ensure_daemon_reason: str | None = None) -> None:
         while True:
             wait_for_turn()
-            exit_code = run_runner_worker(list(args), name=f"worker_runner_update_{os.getpid()}")
+            if ensure_daemon_reason:
+                ensure_daemon_stack(reason=ensure_daemon_reason)
+            exit_code = run_runner_worker(
+                list(args),
+                name=f"worker_runner_update_{os.getpid()}",
+                keep_running=still_owns_foreground_runner,
+            )
             if exit_code == 0:
                 return
-            if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+            if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
                 continue
             raise SystemExit(exit_code)
 
@@ -298,8 +284,6 @@ def update(
 
     try:
         wait_for_turn()
-        if stop_daemon_stack:
-            stop_existing_supervisord_process()
 
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             while True:
@@ -307,6 +291,9 @@ def update(
                 do_index = index_only or not migrate_only
                 do_run_until_idle = do_migrate or do_index
                 ran_post_migrate_runner = False
+                full_update_empty = False
+                maintenance_work_queued = False
+                runner_work_queued = False
 
                 if do_migrate:
                     if (
@@ -343,6 +330,8 @@ def update(
                         )
                         print_stats(stats)
                         touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                        maintenance_work_queued = stats.get("queued", 0) > 0
+                        runner_work_queued = runner_work_queued or maintenance_work_queued
                     else:
                         stats_combined = {"phase1": {}, "phase2": {}}
 
@@ -359,6 +348,17 @@ def update(
                             wait_for_turn=wait_for_turn,
                         )
                         print_combined_stats(stats_combined)
+                        full_update_empty = (
+                            stats_combined["phase1"].get("processed", 0) == 0 and stats_combined["phase2"].get("snapshots", 0) == 0
+                        )
+                        maintenance_work_queued = any(
+                            (
+                                stats_combined["phase1"].get("queued", 0),
+                                stats_combined["phase2"].get("queued", 0),
+                                stats_combined["phase2"].get("crawls_sealed", 0),
+                            ),
+                        )
+                        runner_work_queued = runner_work_queued or maintenance_work_queued
 
                     if do_run_until_idle:
                         # Filesystem migration is maintenance on existing
@@ -370,86 +370,108 @@ def update(
                         # rows first, runs the targeted plugins, and may leave
                         # the fs_version maintenance tick hidden behind that
                         # plugin work until another update pass.
-                        print("[*] Phase 3: Running filesystem maintenance until idle...")
-                        if is_filtered_update:
+                        if full_update_empty:
+                            print("[*] No snapshots or legacy archive directories found; skipping filesystem maintenance runner.")
+                        elif not maintenance_work_queued:
+                            print("[*] No filesystem maintenance work queued; skipping filesystem maintenance runner.")
+                        else:
+                            print("[*] Phase 3: Running filesystem maintenance until idle...")
+                        if full_update_empty:
+                            pass
+                        elif not maintenance_work_queued:
+                            pass
+                        elif is_filtered_update:
                             if not touched_snapshot_ids:
                                 print("[*] No matching snapshots queued work for the runner.")
                             for snapshot_id in sorted(touched_snapshot_ids):
                                 run_scoped_runner("--snapshot-id", snapshot_id)
                         else:
-                            run_scoped_runner("--maintenance-only")
+                            run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size))
                         ran_post_migrate_runner = True
 
                 if do_index:
-                    ensure_daemon_stack(reason="search indexing")
-                    search_plugins = _get_search_indexing_plugins()
-                    if not search_plugins:
-                        print("[*] No search indexing plugins are available, nothing to backfill.")
+                    if full_update_empty:
+                        print("[*] No snapshots found; skipping search indexing backfill.")
                     else:
-                        snapshots = _build_filtered_snapshots_queryset(
-                            filter_patterns=filter_patterns,
-                            filter_type=filter_type,
-                            status=status,
-                            url__icontains=url__icontains,
-                            url__istartswith=url__istartswith,
-                            tag=tag,
-                            crawl_id=crawl_id,
-                            limit=limit,
-                            sort=sort,
-                            search=search,
-                            before=before,
-                            after=after,
-                            resume=resume,
-                        )
-                        from django.db.models import Exists, OuterRef, Q
-                        from django.utils import timezone
-                        from archivebox.core.models import ArchiveResult, Snapshot
+                        ensure_daemon_stack(reason="search indexing")
+                        search_plugins = _get_search_indexing_plugins()
+                        if not search_plugins:
+                            print("[*] No search indexing plugins are available, nothing to backfill.")
+                        else:
+                            snapshots = _build_filtered_snapshots_queryset(
+                                filter_patterns=filter_patterns,
+                                filter_type=filter_type,
+                                status=status,
+                                url__icontains=url__icontains,
+                                url__istartswith=url__istartswith,
+                                tag=tag,
+                                crawl_id=crawl_id,
+                                limit=limit,
+                                sort=sort,
+                                search=search,
+                                before=before,
+                                after=after,
+                                resume=resume,
+                            )
+                            from django.db.models import Exists, OuterRef, Q
+                            from django.utils import timezone
+                            from archivebox.core.models import ArchiveResult, Snapshot
 
-                        scoped_snapshot_ids = snapshots.order_by().values("id") if is_filtered_update else None
-                        queued_index_results = ArchiveResult.objects.filter(
-                            status=ArchiveResult.StatusChoices.QUEUED,
-                            plugin__in=search_plugins,
-                        )
-                        if scoped_snapshot_ids is not None:
-                            queued_index_results = queued_index_results.filter(snapshot_id__in=scoped_snapshot_ids)
-
-                        if queued_index_results.exists():
-                            now = timezone.now()
-                            queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
-                            snapshots_to_wake = (
-                                Snapshot.objects.filter(
-                                    status__in=(Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED),
-                                )
-                                .annotate(
-                                    has_queued_index_result=Exists(queued_result_for_snapshot),
-                                )
-                                .filter(
-                                    has_queued_index_result=True,
-                                )
-                                .filter(
-                                    Q(retry_at__isnull=True) | Q(retry_at__gt=now),
-                                )
+                            scoped_snapshot_ids = snapshots.order_by().values("id") if is_filtered_update else None
+                            queued_index_results = ArchiveResult.objects.filter(
+                                status=ArchiveResult.StatusChoices.QUEUED,
+                                plugin__in=search_plugins,
                             )
                             if scoped_snapshot_ids is not None:
-                                snapshots_to_wake = snapshots_to_wake.filter(id__in=scoped_snapshot_ids)
-                            woken_count = snapshots_to_wake.update(
-                                retry_at=now,
-                                modified_at=now,
-                            )
-                            print(
-                                "[*] Existing queued search index jobs found; "
-                                f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.",
-                            )
-                        else:
-                            stats = reindex_snapshots(
-                                snapshots,
-                                search_plugins=search_plugins,
-                                batch_size=batch_size,
-                                collect_ids=is_filtered_update,
-                                wait_for_turn=wait_for_turn,
-                            )
-                            print_index_stats(stats)
-                            touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                                queued_index_results = queued_index_results.filter(snapshot_id__in=scoped_snapshot_ids)
+
+                            if queued_index_results.exists():
+                                runner_work_queued = True
+                                now = timezone.now()
+                                queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
+                                snapshots_to_wake = (
+                                    Snapshot.objects.filter(
+                                        status__in=(Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED),
+                                    )
+                                    .annotate(
+                                        has_queued_index_result=Exists(queued_result_for_snapshot),
+                                    )
+                                    .filter(
+                                        has_queued_index_result=True,
+                                    )
+                                    .filter(
+                                        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+                                    )
+                                )
+                                if scoped_snapshot_ids is not None:
+                                    snapshots_to_wake = snapshots_to_wake.filter(id__in=scoped_snapshot_ids)
+                                woken_count = snapshots_to_wake.update(
+                                    retry_at=now,
+                                    modified_at=now,
+                                )
+                                print(
+                                    "[*] Existing queued search index jobs found; "
+                                    f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.",
+                                )
+                            else:
+                                collect_index_ids = (
+                                    is_filtered_update
+                                    or foreground_runner_owner(
+                                        data_dir=CONSTANTS.DATA_DIR,
+                                        exclude_id=command.id,
+                                    )
+                                    is not None
+                                )
+                                stats = reindex_snapshots(
+                                    snapshots,
+                                    search_plugins=search_plugins,
+                                    batch_size=batch_size,
+                                    collect_ids=collect_index_ids,
+                                    wait_for_turn=wait_for_turn,
+                                )
+                                print_index_stats(stats)
+                                touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                                runner_work_queued = runner_work_queued or stats["queued"] > 0
 
                 if do_run_until_idle and (do_index or not ran_post_migrate_runner):
                     # Search/index backfill intentionally queues targeted
@@ -459,14 +481,26 @@ def update(
                     # For a normal unfiltered `archivebox update`, keep the
                     # historical final pass broad enough to resume genuinely
                     # queued/interrupted crawl work after maintenance is done.
-                    print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
-                    if is_filtered_update:
+                    if full_update_empty:
+                        print("[*] No snapshots found; skipping queued/interrupted crawl runner.")
+                    elif not runner_work_queued:
+                        print("[*] No queued/interrupted crawl work found; skipping queued/interrupted crawl runner.")
+                    else:
+                        print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
+                    if full_update_empty:
+                        pass
+                    elif not runner_work_queued:
+                        pass
+                    elif touched_snapshot_ids and is_filtered_update:
                         if not touched_snapshot_ids:
                             print("[*] No matching snapshots queued work for the runner.")
                         for snapshot_id in sorted(touched_snapshot_ids):
                             run_scoped_runner("--snapshot-id", snapshot_id)
                     else:
-                        run_scoped_runner(*(["--maintenance-only"] if index_only or migrate_only else []))
+                        run_scoped_runner(
+                            *(["--maintenance-only", "--maintenance-batch-size", str(batch_size)] if index_only or migrate_only else []),
+                            ensure_daemon_reason="search indexing" if do_index else None,
+                        )
 
                 if not continuous:
                     break
@@ -713,7 +747,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
         "updated_db": 0,
         "queued": 0,
         "sealed": 0,
-        "crawls_queued": 0,
+        "crawls_sealed": 0,
     }
     current_fs_version = Snapshot._fs_current_version()
 
@@ -721,6 +755,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     if resume:
         queryset = queryset.filter(timestamp__lte=resume)
     total = queryset.count()
+    stats["snapshots"] = total
     print(f"[*] Processing {total} snapshots from database (most recent first)...")
 
     def update_in_batches(rows, *, label: str, **updates) -> int:
@@ -819,7 +854,10 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     queue_stale_fs_batch()
 
     now = timezone.now()
-    stats["crawls_queued"] = (
+    # Crawls with no open child snapshots are already finished. Seal them here
+    # instead of waking the foreground runner; otherwise migration/update can
+    # accidentally re-enter full crawl execution for historical rows.
+    stats["crawls_sealed"] = (
         Crawl.objects.filter(
             status__in=Crawl.RUNNABLE_STATES,
         )
@@ -827,11 +865,12 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
             snapshot_set__status__in=Snapshot.OPEN_STATES,
         )
         .update(
-            retry_at=now,
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
             modified_at=now,
         )
     )
-    stats["updated_db"] += stats["crawls_queued"]
+    stats["updated_db"] += stats["crawls_sealed"]
     return stats
 
 
@@ -969,7 +1008,7 @@ Phase 2 (Process DB):
   Updated JSON:     {s2.get("updated_json", 0)}
   Updated DB rows:  {s2.get("updated_db", 0)}
   Sealed snapshots: {s2.get("sealed", 0)}
-  Queued crawls:    {s2.get("crawls_queued", 0)}
+  Sealed crawls:    {s2.get("crawls_sealed", 0)}
 """)
 
 
@@ -987,22 +1026,11 @@ def print_index_stats(stats: dict[str, Any]) -> None:
 
 @click.command()
 @click.option("--resume", type=str, help="Resume from timestamp")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of snapshots to update")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--search", help="Search mode to use for positional query")
-@click.option("--before", type=float, help="Only snapshots before timestamp")
-@click.option("--after", type=float, help="Only snapshots after timestamp")
-@click.option("--filter-type", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact")
 @click.option("--batch-size", type=int, default=500, help="Commit every N records")
 @click.option("--continuous", is_flag=True, help="Run continuously as background worker")
 @click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content")
 @click.option("--migrate-only", is_flag=True, help="Only migrate filesystem and update database/index state")
-@click.argument("filter_patterns", nargs=-1)
+@snapshot_filter_options(default_filter_type="exact")
 @docstring(update.__doc__)
 def main(**kwargs):
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
@@ -1010,6 +1038,8 @@ def main(**kwargs):
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             update(**kwargs)
+    except ValueError as err:
+        raise click.BadParameter(str(err), param_hint="--status") from err
     except KeyboardInterrupt:
         raise SystemExit(130) from None
 

archivebox/cli/archivebox_version.py

@@ -133,7 +133,7 @@ def version(
     try:
         DATA_DIR_STAT = CONSTANTS.DATA_DIR.stat()
         prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
             f"FS_UID={DATA_DIR_STAT.st_uid}:{DATA_DIR_STAT.st_gid}",
             f"FS_PERMS={config.OUTPUT_PERMISSIONS}",
             f"FS_ATOMIC={config.ENFORCE_ATOMIC_WRITES}",
@@ -141,7 +141,7 @@ def version(
         )
     except Exception:
         prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
         )
 
     prnt(

archivebox/config/common.py

@@ -7,10 +7,9 @@ import os
 import re
 import secrets
 import sys
-import time
 import shutil
 import inspect
-from functools import lru_cache, wraps
+from functools import lru_cache
 from collections.abc import Mapping
 from datetime import timedelta
 from typing import Any, ClassVar, cast
@@ -42,24 +41,6 @@ _STDERR_CONSOLE = Console(stderr=True)
 _WARNED_ARCHIVING_CONFIGS: set[tuple[int, bool]] = set()
 
 
-def _perf_trace(label):
-    def decorator(func):
-        @wraps(func)
-        def wrapper(*args, **kwargs):
-            if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
-                return func(*args, **kwargs)
-            started_at = time.perf_counter()
-            try:
-                return func(*args, **kwargs)
-            finally:
-                elapsed_ms = (time.perf_counter() - started_at) * 1000
-                print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
-
-        return wrapper
-
-    return decorator
-
-
 def _legacy_bool(value: object) -> bool | None:
     if value is None:
         return None
@@ -195,6 +176,10 @@ def build_crawl_config_snapshot(
     plugin_owned_keys = set(_plugin_config_properties(PLUGIN_CONFIG_SCHEMAS)) - set(ArchiveBoxBaseConfig.model_fields)
     effective = get_config(persona=persona, base_config=base_config)
     frozen = effective.for_crawl_frozen(persona=persona)
+    for key in ("BIND_ADDR", "BASE_URL", "CSRF_TRUSTED_ORIGINS", "SERVER_SECURITY_MODE"):
+        value = getattr(effective, key, None)
+        if value is not None:
+            frozen[key] = value
     if persona is not None:
         persona_config = persona.get_derived_config()
         for key in plugin_owned_keys - explicit_overrides:
@@ -204,6 +189,10 @@ def build_crawl_config_snapshot(
         resolved = get_config(base_config=frozen, overrides=overrides, include_machine=False)
         resolved_payload = normalize_runtime_config(resolved)
         frozen = resolved.for_crawl_frozen(persona=persona)
+        for key in ("BIND_ADDR", "BASE_URL", "CSRF_TRUSTED_ORIGINS", "SERVER_SECURITY_MODE"):
+            value = getattr(resolved, key, None)
+            if value is not None:
+                frozen[key] = value
         for key in plugin_owned_keys & explicit_overrides:
             if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_FROZEN and key in resolved_payload:
                 frozen[key] = resolved_payload[key]
@@ -257,7 +246,7 @@ class StorageConfig(BaseConfigSet):
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     # TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
-    # must be a short path due to unix path length restrictions for socket files (<100 chars)
+    # must be a short path due to unix path length restrictions for socket files (<90 chars)
     # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
     TMP_DIR: Path = Field(default=CONSTANTS.DEFAULT_TMP_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
@@ -266,10 +255,6 @@ class StorageConfig(BaseConfigSet):
     # should not be a remote/network/FUSE mount for speed reasons, otherwise extractors will be slow
     LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
-    # LIB_BIN_DIR is an optional human-facing symlink convenience directory.
-    # Runtime lookup must use provider-specific paths under LIB_DIR instead.
-    LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-
     OUTPUT_PERMISSIONS: str = Field(default="644")
     ENFORCE_ATOMIC_WRITES: bool = Field(default=True)
     ALLOW_NO_UNIX_SOCKETS: bool = Field(default=False, alias="ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS")
@@ -664,8 +649,10 @@ class ArchiveBoxBaseConfig(
             "ACTIVE_PERSONA",
             "CHROME_DOWNLOADS_DIR",
             "CHROME_USER_DATA_DIR",
+            "CRAWL_DIR",
             "DEFAULT_PERSONA",
             "EXTRA_CONTEXT",
+            "SNAP_DIR",
         }
         return frozenset(
             key for key, scope in cls._scope_by_key().items() if scope == _SCOPE_CRAWL_EXECUTION and key in runtime_derived_keys
@@ -678,7 +665,10 @@ class ArchiveBoxBaseConfig(
 
     def for_crawl(self) -> dict[str, Any]:
         """Config scoped to crawl execution, without runtime object overlays."""
-        return self._scoped_config(include_execution=True)
+        config = self._scoped_config(include_execution=True)
+        for key in type(self).runtime_derived_config_keys():
+            config.pop(key, None)
+        return config
 
     def for_crawl_frozen(self, *, persona: Any = None) -> dict[str, Any]:
         """Config safe to persist permanently on Crawl.config."""
@@ -691,7 +681,6 @@ class ArchiveBoxBaseConfig(
                     frozen.pop(key, None)
         return frozen
 
-    @_perf_trace("archivebox.config.for_crawl_runtime")
     def for_crawl_runtime(
         self,
         *,
@@ -705,10 +694,9 @@ class ArchiveBoxBaseConfig(
     ) -> dict[str, Any]:
         """Config payload safe to pass to crawl/snapshot hook execution."""
         config = self.for_crawl()
+        config["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
         scope_by_key = type(self)._scope_by_key()
         model_fields = type(self).model_fields
-        for key in type(self).runtime_derived_config_keys():
-            config.pop(key, None)
         # ArchiveBox owns SEARCH_BACKEND_ENGINE and uses it during model
         # validation to derive the selected backend's *_ENABLED flag. Hooks
         # only receive the backend-local flags, never the selector itself.
@@ -752,13 +740,6 @@ class ArchiveBoxBaseConfig(
             lib_dir = CONSTANTS.DATA_DIR / lib_dir
         self.LIB_DIR = lib_dir.resolve()
 
-        lib_bin_dir = self.LIB_BIN_DIR.expanduser()
-        if lib_bin_dir == CONSTANTS.DEFAULT_LIB_BIN_DIR and self.LIB_DIR != CONSTANTS.DEFAULT_LIB_DIR:
-            lib_bin_dir = self.LIB_DIR / "bin"
-        elif not lib_bin_dir.is_absolute():
-            lib_bin_dir = CONSTANTS.DATA_DIR / lib_bin_dir
-        self.LIB_BIN_DIR = lib_bin_dir.resolve()
-
         return self
 
     @model_validator(mode="after")
@@ -942,14 +923,11 @@ def find_config_type(key: str) -> str:
 
 def find_config_source(key: str, merged_config: Mapping[str, Any]) -> str:
     """Determine where a config value comes from."""
-    try:
-        from archivebox.machine.models import Machine
+    from archivebox.machine.models import Machine
 
-        machine = Machine.current()
-        if machine.config and key in machine.config:
-            return "Machine"
-    except Exception:
-        pass
+    machine = Machine.current()
+    if machine.config and key in machine.config:
+        return "Machine"
 
     if key in os.environ:
         return "Environment"
@@ -976,7 +954,6 @@ def get_request_config(request: Any, *, resolve_plugins: bool = False) -> Archiv
     return request_config
 
 
-@_perf_trace("archivebox.config.get_config")
 def get_config(
     defaults: ConfigOverrides | None = None,
     overrides: ConfigOverrides | None = None,
@@ -1157,7 +1134,6 @@ def get_config(
             if is_sensitive_config_key(key) and value not in (None, ""):
                 setattr(config, key, SENSITIVE_CONFIG_VALUE_REDACTED)
     os.environ["LIB_DIR"] = str(config.LIB_DIR)
-    os.environ["LIB_BIN_DIR"] = str(config.LIB_BIN_DIR)
     os.environ["ABXPKG_LIB_DIR"] = str(config.LIB_DIR)
     archiving_warning_key = (config.TIMEOUT, config.USE_COLOR)
     if archiving_warning_key not in _WARNED_ARCHIVING_CONFIGS:

archivebox/config/constants.py

@@ -36,8 +36,8 @@ from .permissions import (
     IN_DOCKER,
     RUNNING_AS_UID,
     RUNNING_AS_GID,
-    DEFAULT_PUID,
-    DEFAULT_PGID,
+    DEFAULT_UID,
+    DEFAULT_GID,
     ARCHIVEBOX_USER,
     ARCHIVEBOX_GROUP,
 )
@@ -66,8 +66,8 @@ class ConstantsDict:
     ARCHIVEBOX_GROUP: int = ARCHIVEBOX_GROUP
     RUNNING_AS_UID: int = RUNNING_AS_UID
     RUNNING_AS_GID: int = RUNNING_AS_GID
-    DEFAULT_PUID: int = DEFAULT_PUID
-    DEFAULT_PGID: int = DEFAULT_PGID
+    DEFAULT_UID: int = DEFAULT_UID
+    DEFAULT_GID: int = DEFAULT_GID
     IS_INSIDE_VENV: bool = sys.prefix != sys.base_prefix
 
     # Source code dirs
@@ -118,7 +118,6 @@ class ConstantsDict:
         "LIB_DIR",
         user_config_path("abx") / LIB_DIR_NAME,
     )
-    DEFAULT_LIB_BIN_DIR: Path = _env_path("LIB_BIN_DIR", DEFAULT_LIB_DIR / "bin")
 
     RESERVED_ARCHIVE_DIR_NAMES: frozenset[str] = frozenset(
         (

archivebox/config/paths.py

@@ -20,6 +20,8 @@ if TYPE_CHECKING:
 
 PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent  # archivebox source code dir
 DATA_DIR: Path = Path(os.getcwd()).resolve()  # archivebox user data dir
+MAX_TMP_SOCKET_URL_LENGTH = 90
+SUPERVISORD_SOCKET_FILENAME = "supervisord.sock"
 
 
 def _env_path(key: str, default: Path) -> Path:
@@ -120,7 +122,7 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No
     try:
         with SudoPermission(uid=uid, fallback=fallback):
             test_file.exists()
-            test_file.write_text(f"Checking if PUID={uid} PGID={gid} can write to dir")
+            test_file.write_text(f"Checking if uid={uid} gid={gid} can write to dir")
             test_file.unlink()
             return True
     except (OSError, PermissionError):
@@ -176,33 +178,45 @@ def create_and_chown_dir(dir_path: Path) -> None:
 
 
 def tmp_dir_socket_path_is_short_enough(dir_path: Path) -> bool:
-    socket_file = dir_path.absolute().resolve() / "supervisord.sock"
-    return len(f"file://{socket_file}") <= 96
+    socket_file = dir_path.absolute().resolve() / SUPERVISORD_SOCKET_FILENAME
+    return len(f"file://{socket_file}") < MAX_TMP_SOCKET_URL_LENGTH
+
+
+def tmp_dir_candidates(config: "ArchiveBoxConfig") -> list[Path]:
+    from archivebox.config.constants import CONSTANTS
+
+    collection_id = get_collection_id()
+    collection_id_short = collection_id[:4]
+    system_tmp_dir = Path(tempfile.gettempdir())
+    candidates = [
+        config.TMP_DIR,  # <user-specified>
+        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
+        Path("/var/run/archivebox") / collection_id,
+        Path("/tmp") / "archivebox" / collection_id,
+        Path("~/.tmp/archivebox").expanduser() / collection_id,
+        system_tmp_dir / "archivebox" / collection_id,
+        system_tmp_dir / "archivebox" / collection_id_short,
+        system_tmp_dir / "abx" / collection_id_short,
+    ]
+    seen = set()
+    unique_candidates = []
+    for path in candidates:
+        path_key = str(path.expanduser().absolute())
+        if path_key in seen:
+            continue
+        seen.add(path_key)
+        unique_candidates.append(path)
+    return unique_candidates
 
 
 def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
-    from archivebox.config.constants import CONSTANTS
     from archivebox.config.common import get_config
     from archivebox.misc.checks import check_tmp_dir
 
     config = config or get_config(**config_kwargs)
-    # try a few potential directories in order of preference
-    CANDIDATES = [
-        config.TMP_DIR,  # <user-specified>
-        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
-        Path("/var/run/archivebox") / get_collection_id(),  # /var/run/archivebox/abc5d8512
-        Path("/tmp") / "archivebox" / get_collection_id(),  # /tmp/archivebox/abc5d8512
-        Path("~/.tmp/archivebox").expanduser() / get_collection_id(),  # ~/.tmp/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id(),  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d
-        Path(tempfile.gettempdir()) / "abx" / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/abx/abc5
-    ]
+    candidates = tmp_dir_candidates(config)
     fallback_candidate = None
-    for candidate in CANDIDATES:
+    for candidate in candidates:
         try:
             create_and_chown_dir(candidate)
         except Exception:
@@ -231,7 +245,7 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxC
         return fallback_candidate
 
     if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {CANDIDATES}!")
+        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {candidates}!")
 
 
 def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@@ -329,7 +343,10 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
             "TMP_DIR": {
                 "path": tmp_dir.resolve(),
                 "enabled": True,
-                "is_valid": os.path.isdir(tmp_dir) and os.access(tmp_dir, os.R_OK) and os.access(tmp_dir, os.W_OK),  # read + write
+                "is_valid": os.path.isdir(tmp_dir)
+                and os.access(tmp_dir, os.R_OK)
+                and os.access(tmp_dir, os.W_OK)
+                and tmp_dir_socket_path_is_short_enough(tmp_dir),
             },
             # "CACHE_DIR": {
             #     "path": CACHE_DIR.resolve(),
@@ -351,8 +368,6 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
     except Exception:
         lib_dir = config.LIB_DIR
 
-    lib_bin_dir = lib_dir / "bin"
-
     return AttrDict(
         {
             "PACKAGE_DIR": {
@@ -380,12 +395,5 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 "enabled": True,
                 "is_valid": os.path.isdir(lib_dir) and os.access(lib_dir, os.R_OK) and os.access(lib_dir, os.W_OK),  # read + write
             },
-            "LIB_BIN_DIR": {
-                "path": lib_bin_dir.resolve(),
-                "enabled": True,
-                "is_valid": os.path.isdir(lib_bin_dir)
-                and os.access(lib_bin_dir, os.R_OK)
-                and os.access(lib_bin_dir, os.W_OK),  # read + write
-            },
         },
     )

archivebox/config/permissions.py

@@ -24,8 +24,8 @@ except PermissionError:
     DATA_DIR_UID = 0
     DATA_DIR_GID = 0
 
-DEFAULT_PUID = 911
-DEFAULT_PGID = 911
+DEFAULT_UID = 911
+DEFAULT_GID = 911
 RUNNING_AS_UID = os.getuid()
 RUNNING_AS_GID = os.getgid()
 EUID = os.geteuid()
@@ -42,23 +42,20 @@ IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE",
 
 FALLBACK_UID = RUNNING_AS_UID or SUDO_UID
 FALLBACK_GID = RUNNING_AS_GID or SUDO_GID
-if RUNNING_AS_UID == 0:
-    try:
-        # if we are running as root it's really hard to figure out what the correct archivebox user should be
-        # as a last resort instead of setting DATA_DIR ownership to 0:0 (which breaks it for non-root users)
-        # check if 911:911 archivebox user exists on host system, and use it instead of 0
-        if pwd.getpwuid(DEFAULT_PUID).pw_name == "archivebox":
-            FALLBACK_UID = DEFAULT_PUID
-            FALLBACK_GID = DEFAULT_PGID
-    except Exception:
-        pass
+try:
+    ARCHIVEBOX_ACCOUNT = pwd.getpwnam("archivebox")
+except KeyError:
+    ARCHIVEBOX_ACCOUNT = None
 
-
-os.environ.setdefault("PUID", str(DATA_DIR_UID or EUID or RUNNING_AS_UID or FALLBACK_UID))
-os.environ.setdefault("PGID", str(DATA_DIR_GID or EGID or RUNNING_AS_GID or FALLBACK_GID))
-
-ARCHIVEBOX_USER = int(os.environ["PUID"])
-ARCHIVEBOX_GROUP = int(os.environ["PGID"])
+if DATA_DIR_UID != 0:
+    ARCHIVEBOX_USER = DATA_DIR_UID
+    ARCHIVEBOX_GROUP = DATA_DIR_GID
+elif RUNNING_AS_UID == 0 and ARCHIVEBOX_ACCOUNT is not None:
+    ARCHIVEBOX_USER = ARCHIVEBOX_ACCOUNT.pw_uid
+    ARCHIVEBOX_GROUP = ARCHIVEBOX_ACCOUNT.pw_gid
+else:
+    ARCHIVEBOX_USER = EUID or RUNNING_AS_UID or FALLBACK_UID
+    ARCHIVEBOX_GROUP = EGID or RUNNING_AS_GID or FALLBACK_GID
 if not USER:
     try:
         # alternative method 1 to get username
@@ -94,13 +91,12 @@ except Exception:
 
 
 def drop_privileges():
-    """If running as root, drop privileges to the user that owns the data dir (or PUID)"""
+    """If running as root, drop privileges to the data dir owner or archivebox user."""
 
-    # always run archivebox as the user that owns the data dir, never as root
+    # Always run ArchiveBox as the user that owns the data dir, or as the
+    # archivebox service account when the data dir is root-owned.
     if os.getuid() == 0:
-        # drop permissions to the user that owns the data dir / provided PUID
         if os.geteuid() != ARCHIVEBOX_USER and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
-            # drop our effective UID to the archivebox user's UID
             os.seteuid(ARCHIVEBOX_USER)
 
             # update environment variables so that subprocesses dont try to write to /root

archivebox/config/version.py

@@ -6,6 +6,7 @@ import importlib.metadata
 from pathlib import Path
 from functools import cache
 from datetime import datetime
+import re
 
 #############################################################################################
 
@@ -41,11 +42,57 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR):
 
 @cache
 def get_COMMIT_HASH() -> str | None:
+    for env_var in ("ARCHIVEBOX_COMMIT_HASH", "COMMIT_HASH"):
+        env_commit_hash = os.environ.get(env_var, "").strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", env_commit_hash):
+            return env_commit_hash
+
+    if IN_DOCKER:
+        try:
+            version_txt = Path("/VERSION.txt").read_text()
+            docker_commit_hashes = re.findall(r"COMMIT_HASH=([0-9a-fA-F]{40})", version_txt)
+            if docker_commit_hashes:
+                return docker_commit_hashes[-1]
+        except Exception:
+            pass
+
+    def _read_git_file(git_dir: Path, ref: str) -> str | None:
+        try:
+            return git_dir.joinpath(ref).read_text().strip()
+        except Exception:
+            pass
+
+        try:
+            packed_refs = git_dir.joinpath("packed-refs").read_text().splitlines()
+        except Exception:
+            return None
+
+        for line in packed_refs:
+            if line.startswith("#") or line.startswith("^") or not line.strip():
+                continue
+            commit_hash, packed_ref = line.split(" ", 1)
+            if packed_ref == ref:
+                return commit_hash.strip()
+
+        return None
+
     try:
         git_dir = PACKAGE_DIR.parent / ".git"
-        ref = (git_dir / "HEAD").read_text().strip().split(" ")[-1]
-        commit_hash = git_dir.joinpath(ref).read_text().strip()
-        return commit_hash
+        if git_dir.is_file():
+            gitdir_line = git_dir.read_text().strip()
+            gitdir_path = gitdir_line.removeprefix("gitdir:").strip()
+            git_dir = Path(gitdir_path)
+            if not git_dir.is_absolute():
+                git_dir = PACKAGE_DIR.parent / git_dir
+
+        head = (git_dir / "HEAD").read_text().strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", head):
+            return head
+
+        ref = head.removeprefix("ref:").strip()
+        commit_hash = _read_git_file(git_dir, ref)
+        if commit_hash:
+            return commit_hash
     except Exception:
         pass
 

archivebox/config/views.py

@@ -18,7 +18,6 @@ from archivebox.misc.util import parse_date
 
 from archivebox.machine.models import Binary
 
-LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/"
 INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"
 
@@ -32,10 +31,6 @@ def format_parsed_datetime(value: object) -> str:
     return parsed.strftime("%Y-%m-%d %H:%M:%S") if parsed else ""
 
 
-def get_live_config_url(key: str) -> str:
-    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
-
-
 def get_environment_binary_url(name: str) -> str:
     return f"{ENVIRONMENT_BINARIES_BASE_URL}{quote(name)}/"
 

archivebox/core/admin_archiveresults.py

@@ -75,6 +75,12 @@ def get_plugin_admin_url(plugin_name: str) -> str:
     return f"{LIVE_PLUGIN_BASE_URL}builtin.{quote(plugin_name)}/"
 
 
+def get_process_link_label(process) -> str:
+    if process.pid:
+        return str(process.pid)
+    return str(process.id)[-8:]
+
+
 def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
     """Render a nice inline list view of archive results with status, plugin, output, and actions."""
 
@@ -139,7 +145,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
             process_display = f'''
                 <a href="{reverse("admin:machine_process_change", args=[process.id])}"
                    style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
-                   title="View process">{process.pid or "-"}</a>
+                   title="View process">{get_process_link_label(process)}</a>
             '''
 
         machine_display = "-"
@@ -517,7 +523,6 @@ class ArchiveResultAdmin(BaseModelAdmin):
             super()
             .get_queryset(request)
             .defer(
-                "config",
                 "notes",
                 "output_json",
             )
@@ -660,7 +665,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
         process = result.process_record
         if not process:
             return "-"
-        process_label = process.pid or "-"
+        process_label = get_process_link_label(process)
         return format_html(
             '<a href="{}"><code>{}</code></a>',
             reverse("admin:machine_process_change", args=[process.id]),
@@ -674,9 +679,8 @@ class ArchiveResultAdmin(BaseModelAdmin):
             return "-"
         machine = process.machine
         return format_html(
-            '<a href="{}"><code>{}</code> {}</a>',
+            '<a href="{}">{}</a>',
             reverse("admin:machine_machine_change", args=[machine.id]),
-            str(machine.id)[:8],
             machine.hostname,
         )
 

archivebox/core/admin_snapshots.py

@@ -30,7 +30,9 @@ from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_p
 from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 
 from archivebox.core.models import Tag, Snapshot, ArchiveResult
+from archivebox.crawls.models import Crawl
 from archivebox.core.admin_archiveresults import render_archiveresults_list
+from archivebox.core.preview_util import EXTENSION_SCREENSHOT_PLUGIN
 from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
@@ -107,9 +109,7 @@ class SnapshotPermissionsListFilter(admin.SimpleListFilter):
     def queryset(self, request, queryset):
         value = self.value()
         if value:
-            return queryset.filter(
-                Q(permissions=value) | (Q(permissions__isnull=True) & Q(crawl__permissions=value)),
-            )
+            return queryset.filter(permissions=value)
         return queryset
 
 
@@ -229,8 +229,6 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
                         total_results=ArchiveResult.snapshot_count_expr(),
                         matching_results=ArchiveResult.snapshot_count_expr(status=status),
                     ).filter(matching_results__gt=F("total_results") / 2)
-                    queryset._archivebox_count_hint = "model_estimate"
-                    queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
                     return queryset
 
                 # Rare statuses are faster status-first: use the
@@ -277,13 +275,7 @@ class SnapshotChangeList(SearchResultsChangeList):
         super().get_results(request)
         if request.GET.get("_embedded") == "crawl":
             self.full_result_count = self.result_count
-        else:
-            self.full_result_count = self.model_admin.get_paginator(
-                request,
-                self.model._default_manager.all().order_by(),
-                self.list_per_page,
-            ).count
-        self.show_full_result_count = True
+            self.show_full_result_count = True
         self._attach_archiveresult_summaries()
 
 
@@ -320,10 +312,7 @@ class SnapshotAdminForm(forms.ModelForm):
         instance = super().save(commit=False)
         permissions = self.cleaned_data["permissions_config"]
         config = dict(instance.config or {})
-        if permissions == instance.crawl.permissions:
-            config.pop("PERMISSIONS", None)
-        else:
-            config["PERMISSIONS"] = permissions
+        config["PERMISSIONS"] = permissions
         instance.config = config
 
         # Handle tags_editor field
@@ -465,7 +454,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         ),
     )
 
-    ordering = ["-timestamp"]
+    ordering = ["-created_at"]
     actions = [
         "add_tags",
         "remove_tags",
@@ -482,7 +471,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     paginator = AcceleratedPaginator
 
     save_on_top = True
-    show_full_result_count = False
+    show_full_result_count = True
 
     def get_changelist(self, request, **kwargs):
         return SnapshotChangeList
@@ -578,12 +567,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         if permissions not in dict(PERMISSIONS_CHOICES):
             return HttpResponseBadRequest("Invalid permissions value")
 
-        snapshot = get_object_or_404(Snapshot.objects.select_related("crawl"), pk=object_id)
+        snapshot = get_object_or_404(Snapshot, pk=object_id)
         config = dict(snapshot.config or {})
-        if permissions == snapshot.crawl.permissions:
-            config.pop("PERMISSIONS", None)
-        else:
-            config["PERMISSIONS"] = permissions
+        config["PERMISSIONS"] = permissions
 
         # Keep the quick-edit write to one targeted UPDATE so SQLite only holds
         # the write lock for the permission/config change itself. safe_update()
@@ -606,18 +592,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         now = timezone.now()
         updated = 0
         batch = []
-        snapshots = (
-            queryset.select_related(None)
-            .select_related("crawl")
-            .only("id", "config", "crawl__id", "crawl__permissions")
-            .prefetch_related(None)
-        )
+        snapshots = queryset.select_related(None).only("id", "config").prefetch_related(None)
         for snapshot in snapshots.iterator(chunk_size=500):
             config = dict(snapshot.config or {})
-            if permissions == snapshot.crawl.permissions:
-                config.pop("PERMISSIONS", None)
-            else:
-                config["PERMISSIONS"] = permissions
+            config["PERMISSIONS"] = permissions
             snapshot.config = config
             snapshot.modified_at = now
             batch.append(snapshot)
@@ -665,13 +643,26 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                         "plugin",
                         "status",
                         "output_size",
+                        "output_files",
+                    ),
+                ),
+            )
+        else:
+            prefetches.append(
+                Prefetch(
+                    "crawl",
+                    queryset=Crawl.objects.select_related("created_by").only(
+                        "id",
+                        "created_by_id",
+                        "created_by__id",
+                        "created_by__username",
                     ),
                 ),
             )
 
-        qs = super().get_queryset(request).select_related("crawl__created_by")
+        qs = super().get_queryset(request)
         if is_change_view:
-            qs = qs.defer("notes")
+            qs = qs.select_related("crawl__created_by").defer("notes")
         else:
             qs = qs.only(
                 "id",
@@ -685,13 +676,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                 "fs_version",
                 "output_size",
                 "permissions",
-                "crawl__id",
-                "crawl__permissions",
-                "crawl__persona_id",
-                "crawl__status",
-                "crawl__created_by_id",
-                "crawl__created_by__id",
-                "crawl__created_by__username",
             )
         qs = qs.prefetch_related(*prefetches)
         if needs_files_sort:
@@ -707,10 +691,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def permissions_badge(self, obj):
         permissions = obj.__dict__.get("snapshot_permissions")
         if permissions is None:
-            if obj.permissions:
-                permissions = obj.permissions
-            else:
-                permissions = obj.crawl.permissions
+            permissions = obj.permissions
         permissions = normalize_permissions(permissions)
         icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
         menu_items = format_html_join(
@@ -961,21 +942,29 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         if results is not None:
             has_screenshot = any(r.plugin == "screenshot" for r in results)
             has_favicon = any(r.plugin == "favicon" for r in results)
+            has_extension_screenshot = any(r.plugin == EXTENSION_SCREENSHOT_PLUGIN for r in results)
         else:
-            available_plugins = set(obj.archiveresult_set.filter(plugin__in=("screenshot", "favicon")).values_list("plugin", flat=True))
+            available_plugins = set(
+                obj.archiveresult_set.filter(plugin__in=("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")).values_list(
+                    "plugin",
+                    flat=True,
+                ),
+            )
             has_screenshot = "screenshot" in available_plugins
             has_favicon = "favicon" in available_plugins
+            has_extension_screenshot = EXTENSION_SCREENSHOT_PLUGIN in available_plugins
 
-        if not has_screenshot and not has_favicon:
+        extension_screenshot_urls = [
+            build_snapshot_url(str(obj.id), f"{EXTENSION_SCREENSHOT_PLUGIN}/screenshot-1.png", request=request, config=config),
+            build_snapshot_url(str(obj.id), f"{EXTENSION_SCREENSHOT_PLUGIN}/screenshot.png", request=request, config=config),
+        ]
+
+        if not has_screenshot and not has_extension_screenshot and not has_favicon:
             return None
 
-        if has_screenshot:
+        if has_screenshot or has_extension_screenshot:
             img_url = build_snapshot_url(str(obj.id), "screenshot/screenshot.png", request=request, config=config)
-            fallbacks = [
-                build_snapshot_url(str(obj.id), "screenshot.png", request=request, config=config),
-                build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config),
-                build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config),
-            ]
+            fallbacks = extension_screenshot_urls
             img_alt = "Screenshot"
             preview_class = "screenshot"
         else:

archivebox/core/admin_tags.py

@@ -117,7 +117,7 @@ class TagAdmin(BaseModelAdmin):
         sort = normalize_tag_sort((request.GET.get("sort") or "created_desc").strip())
         created_by = normalize_created_by_filter((request.GET.get("created_by") or "").strip())
         year = normalize_created_year_filter((request.GET.get("year") or "").strip())
-        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "yes").strip())
+        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "all").strip())
         context = {
             **self.admin_site.each_context(request),
             **(extra_context or {}),

archivebox/core/forms.py

@@ -41,11 +41,12 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             attrs={
                 "data-url-regex": URL_REGEX.pattern,
                 "placeholder": (
-                    "Enter URLs to archive, as one per line, CSV, JSON, or embedded in text "
-                    "(e.g. markdown, HTML, etc.). Examples:\n"
-                    "https://example.com\n"
-                    "https://news.ycombinator.com,https://news.google.com\n"
-                    "[ArchiveBox](https://github.com/ArchiveBox/ArchiveBox)"
+                    "\n",
+                    "Enter URL(s) to archive. Any format is ok: one per line, CSV, JSON, embedded in text, etc."
+                    "Examples:\n\n"
+                    "https://example.com\n\n"
+                    "https://news.ycombinator.com,https://news.google.com\n\n"
+                    "Or any text-based content [containing URLs](https://github.com/ArchiveBox/ArchiveBox)...",
                 ),
             },
         ),
@@ -314,12 +315,9 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 
     def clean_url(self):
         value = self.cleaned_data.get("url") or ""
-        valid_urls = []
-        for url in find_all_urls(value):
-            valid_urls.append(url)
-        if not valid_urls:
+        if not list(find_all_urls(value)):
             raise forms.ValidationError("Enter at least one valid URL.")
-        return "\n".join(valid_urls)
+        return value
 
     def clean_url_filters(self):
         from archivebox.crawls.models import Crawl

archivebox/core/middleware.py

@@ -3,6 +3,7 @@ __package__ = "archivebox.core"
 import ipaddress
 import re
 from pathlib import Path
+from django.conf import settings
 from django.utils import timezone
 from django.contrib.auth.middleware import RemoteUserMiddleware
 from django.contrib.auth.models import AnonymousUser
@@ -24,7 +25,6 @@ from archivebox.core.routes_util import (
     get_base_host,
     get_listen_host,
     get_listen_subdomain,
-    get_public_host,
     get_web_host,
     host_matches,
     is_snapshot_subdomain,
@@ -42,7 +42,7 @@ def _admin_login_hint_cookie_domain(config) -> str | None:
     NOTE: this cookie carries only the single bit "user is logged in on
     admin somewhere"; it MUST NOT be confused with the session cookie,
     which stays admin-host-scoped (see core/settings.py
-    SESSION_COOKIE_DOMAIN comment — admin/public is a security boundary).
+    SESSION_COOKIE_DOMAIN comment — admin/web is a security boundary).
 
     Returns the hostname portion of ``get_base_host`` (which respects
     ``BASE_URL`` and falls back to the local-bind mapping). Strips the
@@ -85,6 +85,34 @@ def TimezoneMiddleware(get_response):
     return middleware
 
 
+def AdminCookieIsolationMiddleware(get_response):
+    def middleware(request):
+        response = get_response(request)
+
+        config = request.__dict__.get("archivebox_config")
+        if config is None:
+            config = get_config(resolve_plugins=False)
+            request.archivebox_config = config
+        if not config.USES_SUBDOMAIN_ROUTING:
+            return response
+
+        request_host = (request.get_host() or "").lower()
+        if host_matches(request_host, get_admin_host(config=config)):
+            return response
+
+        if host_matches(request_host, get_web_host(config=config)):
+            for cookie_name in tuple(response.cookies.keys()):
+                if cookie_name != ADMIN_LOGIN_HINT_COOKIE:
+                    response.cookies.pop(cookie_name, None)
+            return response
+
+        response.cookies.pop(settings.SESSION_COOKIE_NAME, None)
+        response.cookies.pop(settings.CSRF_COOKIE_NAME, None)
+        return response
+
+    return middleware
+
+
 def CacheControlMiddleware(get_response):
     snapshot_path_re = re.compile(r"^/[^/]+/\\d{8}/[^/]+/[0-9a-fA-F-]{8,36}/")
     static_cache_key = (get_COMMIT_HASH() or VERSION or "dev").strip()
@@ -176,7 +204,6 @@ def HostRoutingMiddleware(get_response):
         admin_host = get_admin_host(config=config)
         web_host = get_web_host(config=config)
         api_host = get_api_host(config=config)
-        public_host = get_public_host(config=config)
         listen_host = get_listen_host(config=config)
         subdomain = get_listen_subdomain(request_host, config=config)
 
@@ -185,25 +212,7 @@ def HostRoutingMiddleware(get_response):
         if request.path.startswith("/static/") or request.path in {"/favicon.ico", "/robots.txt"}:
             return get_response(request)
 
-        # In subdomain mode with no explicit BASE_URL we can't safely emit
-        # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
-        # request's own Host (via the request-host fallback in get_base_url),
-        # so prepending ``admin.`` to whatever the client sent produces a
-        # redirect chain of ``admin.admin.admin.<host>``. Pass the request
-        # through; the misconfig banner on the rendered page tells the user
-        # to pin BASE_URL so the redirects can resume.
-        if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
-            return get_response(request)
-
-        if config.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host):
-            # ``/add`` is admin-only unless ``PUBLIC_ADD_VIEW`` is on. Without
-            # this redirect, hitting it on public.* falls into AddView's
-            # auth check, bounces through ``/accounts/login/?next=/add/`` →
-            # ``/admin/login/?next=/add/``, and Django admin's LoginView
-            # silently drops ``next`` when the user already has an admin
-            # session — dumping the user on the admin homepage instead of
-            # the add form. Routing the request to admin.* directly lets
-            # AddView run on the host where the session lives.
+        if config.USES_SUBDOMAIN_ROUTING and config.BASE_URL and not host_matches(request_host, admin_host):
             add_should_redirect = not config.PUBLIC_ADD_VIEW and (request.path == "/add" or request.path.startswith("/add/"))
             if (
                 request.path == "/admin"
@@ -217,6 +226,20 @@ def HostRoutingMiddleware(get_response):
                     target = f"{target}?{request.META['QUERY_STRING']}"
                 return redirect(target)
 
+        if subdomain and is_snapshot_subdomain(subdomain):
+            view = SnapshotHostView.as_view()
+            return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
+
+        # In subdomain mode with no explicit BASE_URL we can't safely emit
+        # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
+        # request's own Host (via the request-host fallback in get_base_url),
+        # so prepending ``admin.`` to whatever the client sent produces a
+        # redirect chain of ``admin.admin.admin.<host>``. Pass the request
+        # through; the misconfig banner on the rendered page tells the user
+        # to pin BASE_URL so the redirects can resume.
+        if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
+            return get_response(request)
+
         if not config.USES_SUBDOMAIN_ROUTING:
             if host_matches(request_host, listen_host):
                 return get_response(request)
@@ -282,16 +305,7 @@ def HostRoutingMiddleware(get_response):
             request._cached_user = request.user
             return get_response(request)
 
-        if host_matches(request_host, public_host):
-            if request.COOKIES.get(ADMIN_LOGIN_HINT_COOKIE) == "1" and (request.path == "/public" or request.path.startswith("/public/")):
-                target = build_admin_url("/admin/core/snapshot/", request=request)
-                return redirect(target)
-            return get_response(request)
-
         if subdomain:
-            if is_snapshot_subdomain(subdomain):
-                view = SnapshotHostView.as_view()
-                return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
             view = OriginalDomainHostView.as_view()
             return view(request, domain=subdomain, path=request.path.lstrip("/"))
 

archivebox/core/migrations/0049_alter_snapshot_url.py

@@ -0,0 +1,17 @@
+# Generated by Django 6.0.6 on 2026-06-04 22:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0048_remove_archiveresult_config"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="url",
+            field=models.TextField(db_index=True),
+        ),
+    ]

archivebox/core/migrations/0050_snapshot_permissions_not_null.py

@@ -0,0 +1,69 @@
+import django.db.models.fields.json
+import django.db.models.functions.comparison
+from django.db import migrations, models
+from django.db.models import Q
+
+
+VALID_PERMISSIONS = {"public", "unlisted", "private"}
+BATCH_SIZE = 1000
+
+
+def normalize_permissions(value, default="private"):
+    value = str(value or "").strip().lower()
+    return value if value in VALID_PERMISSIONS else default
+
+
+def hydrate_snapshot_permissions(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    db_alias = schema_editor.connection.alias
+    missing_permissions = Q(permissions__isnull=True) | (Q(permissions__isnull=False) & ~Q(permissions__in=VALID_PERMISSIONS))
+    batch = []
+
+    snapshots = (
+        Snapshot.objects.using(db_alias)
+        .filter(missing_permissions)
+        .select_related("crawl")
+        .only("id", "config", "crawl__permissions")
+        .iterator(chunk_size=BATCH_SIZE)
+    )
+    for snapshot in snapshots:
+        config = dict(snapshot.config or {})
+        config["PERMISSIONS"] = normalize_permissions(snapshot.crawl.permissions)
+        snapshot.config = config
+        batch.append(snapshot)
+        if len(batch) >= BATCH_SIZE:
+            Snapshot.objects.using(db_alias).bulk_update(batch, ["config"], batch_size=BATCH_SIZE)
+            batch.clear()
+
+    if batch:
+        Snapshot.objects.using(db_alias).bulk_update(batch, ["config"], batch_size=BATCH_SIZE)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("crawls", "0016_hydrate_crawl_permissions"),
+        ("core", "0049_alter_snapshot_url"),
+    ]
+
+    operations = [
+        migrations.RunPython(hydrate_snapshot_permissions, migrations.RunPython.noop),
+        migrations.RemoveField(
+            model_name="snapshot",
+            name="permissions",
+        ),
+        migrations.AddField(
+            model_name="snapshot",
+            name="permissions",
+            field=models.GeneratedField(
+                db_index=True,
+                db_persist=True,
+                editable=False,
+                expression=django.db.models.functions.comparison.Coalesce(
+                    django.db.models.fields.json.KeyTextTransform("PERMISSIONS", "config"),
+                    models.Value("private"),
+                    output_field=models.CharField(max_length=16),
+                ),
+                output_field=models.CharField(max_length=16),
+            ),
+        ),
+    ]

archivebox/core/models.py

@@ -1,7 +1,7 @@
 __package__ = "archivebox.core"
 
 from typing import TYPE_CHECKING, Optional, Any
-from collections.abc import Iterable, Sequence
+from collections.abc import Iterable, Mapping, Sequence
 import uuid
 from archivebox.uuid_compat import CompactUUIDField, uuid7
 from datetime import datetime, timedelta
@@ -31,7 +31,6 @@ from archivebox.config import CONSTANTS
 from archivebox.config.common import get_config, rprint
 from archivebox.misc.system import atomic_write
 from archivebox.misc.util import (
-    MAX_URL_LENGTH,
     parse_date,
     domain as url_domain,
     to_json,
@@ -39,7 +38,7 @@ from archivebox.misc.util import (
     urlencode,
     htmlencode,
     urldecode,
-    validate_url_length,
+    validate_url,
 )
 from archivebox.plugins.discovery import (
     get_plugins,
@@ -156,6 +155,32 @@ class SnapshotTag(models.Model):
 class SnapshotQuerySet(models.QuerySet):
     """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
 
+    def bulk_create(self, objs, *args, **kwargs):
+        objs = list(objs)
+        missing_crawl_ids = set()
+        from archivebox.core.permissions import PERMISSIONS_VALUES
+
+        for obj in objs:
+            if isinstance(obj, self.model):
+                config = dict(obj.config or {})
+                permission = str(config.get("PERMISSIONS") or "").strip().lower()
+                if permission not in PERMISSIONS_VALUES and obj.crawl_id:
+                    crawl = getattr(obj, "crawl", None)
+                    if not getattr(crawl, "permissions", None):
+                        missing_crawl_ids.add(str(obj.crawl_id))
+
+        crawl_permissions_by_id = {}
+        if missing_crawl_ids:
+            crawl_permissions_by_id = {
+                str(crawl_id): permissions
+                for crawl_id, permissions in Crawl.objects.filter(pk__in=missing_crawl_ids).values_list("pk", "permissions")
+            }
+
+        for obj in objs:
+            if isinstance(obj, self.model):
+                obj.ensure_permissions_config(crawl_permissions=crawl_permissions_by_id.get(str(obj.crawl_id)))
+        return super().bulk_create(objs, *args, **kwargs)
+
     def paged_iterator(self, chunk_size: int = 500):
         """
         Iterate snapshots using bounded keyset pages instead of one streaming cursor.
@@ -267,6 +292,22 @@ class SnapshotQuerySet(models.QuerySet):
         "tag": lambda pattern: models.Q(tags__name=pattern),
         "timestamp": lambda pattern: models.Q(timestamp=pattern),
     }
+    FILTER_TYPE_CHOICES = tuple(FILTER_TYPES)
+    FILTER_ARG_KEYS = (
+        "after",
+        "before",
+        "filter_type",
+        "filter_patterns",
+        "status",
+        "url__icontains",
+        "url__istartswith",
+        "tag",
+        "crawl_id",
+        "limit",
+        "sort",
+        "search",
+    )
+    SPECIAL_FILTER_ARG_KEYS = frozenset({"filter_patterns", "filter_type", "query", "search", "tag", "before", "after", "limit", "sort"})
 
     def filter_by_patterns(self, patterns: list[str], filter_type: str = "exact") -> "SnapshotQuerySet":
         """Filter snapshots by URL patterns using specified filter type"""
@@ -283,6 +324,62 @@ class SnapshotQuerySet(models.QuerySet):
                 raise SystemExit(2)
         return self.filter(q_filter)
 
+    def search(self, **kwargs) -> "SnapshotQuerySet":
+        from datetime import timezone as dt_timezone
+
+        from archivebox.core.snapshot_status import filter_snapshots_by_status
+        from archivebox.search.query import apply_snapshot_search
+
+        queryset = self
+        filter_patterns = tuple(str(pattern) for pattern in kwargs.get("filter_patterns") or ())
+        filter_type = kwargs.get("filter_type") or "substring"
+        query = kwargs.get("query")
+        if isinstance(query, (list, tuple)):
+            query = " ".join(str(part) for part in query)
+        query = (query or (" ".join(filter_patterns) if kwargs.get("search") else "")).strip()
+
+        field_names = {field.name for field in self.model._meta.get_fields()}
+        field_names.update(field.attname for field in self.model._meta.fields)
+        field_filters = {
+            key: value
+            for key, value in kwargs.items()
+            if value is not None and key not in self.SPECIAL_FILTER_ARG_KEYS and key.split("__", 1)[0] in field_names
+        }
+        status = field_filters.pop("status", None)
+        queryset = filter_snapshots_by_status(queryset, status)
+        if field_filters:
+            queryset = queryset.filter(**field_filters)
+        if kwargs.get("tag"):
+            queryset = queryset.filter(tags__name__iexact=kwargs["tag"])
+        if kwargs.get("before") is not None:
+            queryset = queryset.filter(bookmarked_at__lt=datetime.fromtimestamp(float(kwargs["before"]), tz=dt_timezone.utc))
+        if kwargs.get("after") is not None:
+            queryset = queryset.filter(bookmarked_at__gt=datetime.fromtimestamp(float(kwargs["after"]), tz=dt_timezone.utc))
+
+        if query:
+            queryset = apply_snapshot_search(
+                queryset,
+                query,
+                search_mode=kwargs.get("search"),
+                ordering=("-created_at",) if not kwargs.get("sort") else None,
+                max_results=kwargs.get("limit"),
+                skip_backend_when_metadata_satisfies_limit=True,
+                include_metadata_for_forced_backend=True,
+            )
+        elif filter_patterns:
+            queryset = queryset.filter_by_patterns(list(filter_patterns), filter_type)
+
+        if kwargs.get("sort"):
+            queryset = queryset.order_by(kwargs["sort"])
+        elif not queryset.query.order_by:
+            queryset = queryset.order_by("-created_at")
+
+        limit = kwargs.get("limit")
+        if limit is not None and limit > 0:
+            queryset = queryset[:limit]
+
+        return queryset
+
     # =========================================================================
     # Export Methods
     # =========================================================================
@@ -392,11 +489,16 @@ class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ig
 
 
 class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithStateMachine):
+    INTERNAL_INPUT_URL = "archivebox://internal"
+
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
 
-    url = models.CharField(max_length=MAX_URL_LENGTH, unique=False, db_index=True)  # URLs can appear in multiple crawls
+    # Stored as a variable-length TextField so short URLs don't reserve space and very long
+    # URLs (up to MAX_URL_LENGTH chars, enforced in save()) are supported, while keeping a
+    # normal index so exact, prefix, and substring lookups all stay fast.
+    url = models.TextField(db_index=True)  # URLs can appear in multiple crawls
     timestamp = models.CharField(max_length=32, unique=True, db_index=True, editable=False)
     bookmarked_at = models.DateTimeField(default=timezone.now, db_index=True)
     crawl: Crawl = models.ForeignKey(Crawl, on_delete=models.CASCADE, null=False, related_name="snapshot_set", db_index=True)  # type: ignore[assignment]
@@ -431,8 +533,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     )
     config = models.JSONField(default=dict, null=False, blank=False, editable=True)
     permissions = models.GeneratedField(
-        expression=KT("config__PERMISSIONS"),
-        output_field=models.CharField(max_length=16, null=True),
+        expression=Coalesce(KT("config__PERMISSIONS"), Value("private"), output_field=models.CharField(max_length=16)),
+        output_field=models.CharField(max_length=16, null=False),
         db_persist=True,
         db_index=True,
         editable=False,
@@ -682,7 +784,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         )
 
     @classmethod
-    def is_archivebox_internal_url(cls, url: str) -> bool:
+    def is_archivebox_internal_url(cls, url: str, *, config: Mapping[str, Any] | Any | None = None) -> bool:
         parsed = urlparse((url or "").strip())
         if parsed.scheme not in ("http", "https") or not parsed.hostname:
             return False
@@ -690,24 +792,39 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         from archivebox.core.routes_util import (
             get_admin_host,
             get_api_host,
+            get_base_host,
             get_listen_host,
-            get_public_host,
             get_web_host,
             split_host_port,
         )
 
-        config = get_config()
+        if config is None:
+            config = get_config()
+        elif isinstance(config, Mapping):
+            route_config = config
+
+            class RouteConfig:
+                BIND_ADDR = str(route_config.get("BIND_ADDR") or "")
+                BASE_URL = str(route_config.get("BASE_URL") or "")
+                CSRF_TRUSTED_ORIGINS = str(route_config.get("CSRF_TRUSTED_ORIGINS") or "")
+                SERVER_SECURITY_MODE = str(route_config.get("SERVER_SECURITY_MODE") or "")
+
+                @property
+                def USES_SUBDOMAIN_ROUTING(self) -> bool:
+                    return self.SERVER_SECURITY_MODE == "safe-subdomains-fullreplay"
+
+            config = RouteConfig()
         host = parsed.hostname.lower().strip(".")
         port = str(parsed.port) if parsed.port else None
-        protected_subdomains = {"admin", "web", "api", "public"}
+        protected_subdomains = {"admin", "web", "api"}
         protected_hosts: set[tuple[str, str | None]] = set()
         protected_roots: set[tuple[str, str | None]] = set()
         for host_value in (
             get_listen_host(config=config),
+            get_base_host(config=config),
             get_admin_host(config=config),
             get_web_host(config=config),
             get_api_host(config=config),
-            get_public_host(config=config),
         ):
             if not host_value:
                 continue
@@ -760,17 +877,59 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         return Binary.objects.filter(process_set__archiveresult__snapshot_id=self.id).distinct()
 
+    def ensure_permissions_config(self, crawl_permissions: str | None = None) -> bool:
+        config = dict(self.config or {})
+        permission = str(config.get("PERMISSIONS") or "").strip().lower()
+        from archivebox.core.permissions import PERMISSIONS_PUBLIC, PERMISSIONS_VALUES, normalize_permissions
+
+        if permission not in PERMISSIONS_VALUES:
+            if self.crawl_id:
+                if not crawl_permissions:
+                    crawl_permissions = Crawl.objects.filter(pk=self.crawl_id).values_list("permissions", flat=True).first()
+            config["PERMISSIONS"] = normalize_permissions(
+                crawl_permissions,
+                default=PERMISSIONS_PUBLIC,
+            )
+            self.config = config
+            return True
+        elif config.get("PERMISSIONS") != permission:
+            config["PERMISSIONS"] = permission
+            self.config = config
+            return True
+        return False
+
+    def validate_url_for_archiving(self, *, config: Mapping[str, Any] | Any | None = None) -> None:
+        if self.is_internal_input_url():
+            return
+
+        try:
+            validate_url(self.url or "")
+        except ValueError as err:
+            raise ValidationError({"url": str(err)}) from err
+
+        if self.is_archivebox_internal_url(self.url, config=config):
+            raise ValidationError({"url": "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs."})
+
+    def is_internal_input_url(self) -> bool:
+        return (self.url or "").strip() == self.INTERNAL_INPUT_URL and self.depth == 0 and bool(self.crawl_id)
+
     def save(self, *args, **kwargs):
         update_fields = kwargs.get("update_fields")
         validate_url_field = self._state.adding or update_fields is None or "url" in update_fields
-        if validate_url_field:
-            try:
-                validate_url_length(self.url or "")
-            except ValueError as err:
-                raise ValidationError({"url": str(err)}) from err
+        crawl_config_for_save = None
+        crawl_permissions_for_save = None
+        if self.crawl_id and validate_url_field:
+            crawl_row = Crawl.objects.filter(pk=self.crawl_id).values("config", "permissions").first()
+            if crawl_row:
+                crawl_config_for_save = crawl_row.get("config") or {}
+                crawl_permissions_for_save = crawl_row.get("permissions")
 
-            if self.is_archivebox_internal_url(self.url):
-                raise ValidationError({"url": "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs."})
+        if self.ensure_permissions_config(crawl_permissions=crawl_permissions_for_save):
+            if update_fields is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
+
+        if validate_url_field:
+            self.validate_url_for_archiving(config=crawl_config_for_save if self.crawl_id else None)
 
         if not self.bookmarked_at:
             self.bookmarked_at = self.created_at or timezone.now()
@@ -781,12 +940,13 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             self.title = self._normalize_title_candidate(self.title, snapshot_url=self.url or "") or None
 
         # Migrate filesystem if needed (happens automatically on save)
-        if self.pk and self.fs_migration_needed:
+        existing_snapshot = self.pk and not self._state.adding
+        if existing_snapshot and self.fs_migration_needed:
             self.migrate_filesystem_to_current_version()
             update_fields = kwargs.get("update_fields")
             if update_fields is not None:
                 kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "fs_version", "modified_at"]))
-        elif self.pk:
+        elif existing_snapshot:
             current_dir = self.get_storage_path_for_version(self._fs_current_version())
             source_dir = Path(self.output_dir)
             if source_dir.exists() and source_dir != current_dir and not source_dir.is_symlink():
@@ -799,8 +959,29 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         def finish_snapshot_save():
             self.ensure_legacy_archive_symlink()
             self.ensure_crawl_symlink()
-            crawl = self.crawl
-            if not crawl.url_passes_filters(self.url, snapshot=self):
+            crawl = Crawl.objects.filter(pk=self.crawl_id).first()
+            if crawl is None:
+                return
+            crawl_tag_names = crawl.current_tag_names()
+            if crawl_tag_names:
+                # Snapshots can be created by parser hook side-effect records,
+                # direct ORM creates, or legacy crawl URL expansion. Crawl tags
+                # are user-facing metadata on the whole import, so attach them
+                # at the Snapshot.save() boundary instead of relying on every
+                # caller to remember to duplicate this fanout logic.
+                tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=crawl_tag_names)}
+                missing_tags = [Tag(name=name) for name in crawl_tag_names if name not in tags_by_name]
+                if missing_tags:
+                    Tag.objects.bulk_create(missing_tags, ignore_conflicts=True)
+                    tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=crawl_tag_names)}
+                self.tags.add(*[tag.pk for name in crawl_tag_names if (tag := tags_by_name.get(name))])
+            # Snapshot.save() normally appends newly created URLs to Crawl.urls
+            # so legacy/direct crawls can keep their queue text in sync. For
+            # internal-input crawls that would corrupt the original submitted
+            # import text; parsed URLs are represented by child Snapshot rows.
+            if crawl.has_internal_input_root():
+                return
+            if not crawl.url_passes_filters(self.url, snapshot=self, use_effective_config=False):
                 return
             # Best-effort skip if our URL is already recorded on the crawl;
             # the atomic UPDATE below is what actually prevents clobbering.
@@ -1989,6 +2170,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     @admin.display(description="Tags")
     def tags_str(self, nocache=True) -> str | None:
+        if "_tags_str_cached" in self.__dict__:
+            return self.__dict__["_tags_str_cached"]
         calc_tags_str = lambda: ",".join(sorted(tag.name for tag in self.tags.all()))
         prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
         if "tags" in prefetched_cache:
@@ -2582,15 +2765,18 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                 )
                 rprint(f"[red]⚠️  Snapshot.from_json auto-created new crawl {crawl.id} for url={url}[/red]", file=sys.stderr)
 
-        # Parse tags (accept either a list ["tag1", "tag2"] or a comma-separated string "tag1,tag2")
+        # Parser hooks emit child Snapshot records through this generic
+        # dispatcher after crawling an internal import root. Those child rows
+        # must inherit crawl-level tags just like direct Crawl.urls snapshots;
+        # otherwise `archivebox add --tag ... < bookmarks.html` loses the tag
+        # unless the parser format also happened to provide its own tags.
         tags_raw = record.get("tags", "")
-        tag_list = []
+        tag_list = list(crawl.current_tag_names()) if crawl else []
         if isinstance(tags_raw, list):
-            tag_list = list(dict.fromkeys(tag.strip() for tag in tags_raw if tag.strip()))
+            tag_list.extend(tag.strip() for tag in tags_raw if tag.strip())
         elif tags_raw:
-            tag_list = list(
-                dict.fromkeys(tag.strip() for tag in re.split(config.TAG_SEPARATOR_PATTERN, tags_raw) if tag.strip()),
-            )
+            tag_list.extend(tag.strip() for tag in re.split(config.TAG_SEPARATOR_PATTERN, tags_raw) if tag.strip())
+        tag_list = list(dict.fromkeys(tag_list))
 
         # Check for existing snapshot with same URL in same crawl
         # (URLs can exist in multiple crawls, but should be unique within a crawl)
@@ -2663,7 +2849,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         return snapshot
 
-    def create_pending_archiveresults(self) -> list["ArchiveResult"]:
+    def create_pending_archiveresults(self, hooks: Iterable[tuple[str, str]] | None = None) -> list["ArchiveResult"]:
         """
         Create ArchiveResult records for all enabled hooks.
 
@@ -2674,18 +2860,23 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         Creates one ArchiveResult per hook (not per plugin), with hook_name set.
         This enables step-based execution where all hooks in a step can run in parallel.
         """
-        from archivebox.plugins.hooks import discover_hooks
-        from archivebox.config.common import get_config
+        try:
+            self.validate_url_for_archiving()
+        except ValidationError as err:
+            rprint(f"[yellow][!] Skipping blocked snapshot URL: {(self.url or '')[:120]}... ({err})[/yellow]")
+            return []
 
-        # Get merged config with crawl-specific PLUGINS filter
-        config = get_config(crawl=self.crawl, snapshot=self)
-        hooks = discover_hooks("Snapshot", config=config)
+        if hooks is None:
+            from archivebox.plugins.hooks import discover_hooks
+            from archivebox.config.common import get_config
+
+            # Compatibility path for direct model callers. The runner passes its
+            # abx-dl hook inventory explicitly so queued rows match execution.
+            config = get_config(crawl=self.crawl, snapshot=self)
+            hooks = ((hook_path.parent.name, hook_path.stem) for hook_path in discover_hooks("Snapshot", config=config))
         archiveresults = []
 
-        for hook_path in hooks:
-            hook_name = hook_path.stem  # e.g., 'on_Snapshot__50_wget'
-            plugin = hook_path.parent.name  # e.g., 'wget'
-
+        for plugin, hook_name in hooks:
             # ArchiveResult output is one filesystem directory per plugin hook, so
             # retries must update this row in place instead of creating siblings.
             archiveresult, _created = ArchiveResult.objects.get_or_create(
@@ -3173,7 +3364,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         """Convert to CSV string"""
         data = self.to_dict()
         cols = cols or ["timestamp", "is_archived", "url"]
-        return separator.join(to_json(data.get(col, ""), indent=None).ljust(ljust) for col in cols)
+        invalid_cols = [col for col in dict.fromkeys(cols) if col not in data]
+        if invalid_cols:
+            supported_cols = ", ".join(sorted(data))
+            raise ValueError(f"Invalid CSV field(s): {', '.join(invalid_cols)}\nSupported CSV fields: {supported_cols}")
+        return separator.join(to_json(data[col], indent=None).ljust(ljust) for col in cols)
 
     def write_json_details(self, out_dir: Path | str | None = None) -> None:
         """Write JSON index file for this snapshot to its output directory"""

archivebox/core/permissions.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from django.db.models import Q, QuerySet
+from django.db.models import QuerySet
 from django.http import HttpRequest
 
 PERMISSIONS_PUBLIC = "public"
@@ -32,10 +32,7 @@ def is_admin_user(request: HttpRequest) -> bool:
 
 
 def get_snapshot_permissions(snapshot) -> str:
-    permissions = snapshot.permissions
-    if not permissions:
-        permissions = snapshot.crawl.permissions
-    return normalize_permissions(permissions)
+    return normalize_permissions(snapshot.permissions)
 
 
 def can_view_snapshot(request: HttpRequest, snapshot) -> bool:
@@ -56,10 +53,7 @@ def filter_personas_by_permissions(queryset: QuerySet, allowed_permissions: set[
 def filter_snapshots_by_permissions(queryset: QuerySet, *, direct: bool = False, allowed_permissions: set[str] | None = None) -> QuerySet:
     allowed_permissions = allowed_permissions or ({PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED} if direct else {PERMISSIONS_PUBLIC})
     allowed = sorted(allowed_permissions)
-    return queryset.exclude(
-        (Q(permissions__isnull=False) & ~Q(permissions__in=allowed))
-        | (Q(permissions__isnull=True) & (Q(crawl__permissions__isnull=True) | ~Q(crawl__permissions__in=allowed))),
-    )
+    return queryset.filter(permissions__in=allowed)
 
 
 def public_snapshots_queryset(queryset: QuerySet) -> QuerySet:

archivebox/core/preview_util.py

@@ -0,0 +1,4 @@
+__package__ = "archivebox.core"
+
+
+EXTENSION_SCREENSHOT_PLUGIN = "chrome_extension_screenshot"

archivebox/core/recovery_util.py

@@ -1,13 +1,21 @@
 from __future__ import annotations
 
+from pathlib import Path
+
 from django.utils import timezone
 from rich.console import Console
 
 
+def _is_signal_interrupted_exit(exit_code: int | None) -> bool:
+    return exit_code is not None and (exit_code < 0 or exit_code >= 128)
+
+
 def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int]:
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.services.archive_result_service import _collect_output_metadata
     from archivebox.machine.models import Process
+    from django.core.exceptions import ValidationError
     from django.db.models import Exists, OuterRef, Q, Subquery, Value
     from django.db.models.functions import Coalesce
 
@@ -20,7 +28,9 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
         "crawls_queued_without_retry_at": 0,
         "snapshots_queued_without_retry_at": 0,
         "archiveresults_backoff": 0,
+        "snapshots_queued_plugin_rows_waiting_on_stale_lease": 0,
         "archiveresults_started_without_running_process": 0,
+        "archiveresults_missing_for_orphaned_hook_processes": 0,
         "snapshots_started_without_running_results": 0,
         "crawls_started_with_due_snapshots": 0,
         "crawls_started_waiting_on_future_snapshots": 0,
@@ -73,6 +83,24 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
         retry_at__isnull=True,
     ).update(retry_at=now, modified_at=now)
     cleaned["archiveresults_backoff"] = backoff_results.update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now)
+    # Targeted plugin rows on final/paused Snapshots are scheduled through the
+    # parent Snapshot.retry_at. retry_at=NULL is the normal idle marker for a
+    # sealed Snapshot and must not be interpreted as queued work just because
+    # old/synthetic ArchiveResult rows exist. If takeover kills the runner
+    # after it leases the Snapshot but before queued ArchiveResult rows finish,
+    # the rows remain QUEUED while retry_at sits in the future. Recovery runs
+    # only after this runner has won the single-runner gate, so it can safely
+    # unlock those stale plugin leases for immediate processing instead of
+    # waiting out the previous owner's full lock timeout.
+    queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED)
+    cleaned["snapshots_queued_plugin_rows_waiting_on_stale_lease"] = (
+        Snapshot.objects.filter(
+            id__in=queued_plugin_results.values("snapshot_id"),
+            status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
+        )
+        .filter(retry_at__gt=now)
+        .update(retry_at=now, modified_at=now)
+    )
     # Impossible state repair: STARTED ArchiveResults without a live Process
     # have no owner left to emit completion. Requeue only the result row; the
     # snapshot/crawl schedulers will pick up normal retry processing.
@@ -81,6 +109,76 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
         process=None,
         modified_at=now,
     )
+    orphaned_hook_processes = Process.objects.filter(
+        process_type=Process.TypeChoices.HOOK,
+        archiveresult__isnull=True,
+    ).exclude(status=Process.StatusChoices.RUNNING)
+    for process in orphaned_hook_processes.only("id", "pwd", "cmd", "process_type", "status"):
+        hook_script_name = process.hook_script_name
+        if not hook_script_name or not process.pwd:
+            continue
+        plugin_dir = Path(process.pwd)
+        try:
+            # Old or synthetic hook Process rows can point at arbitrary paths.
+            # Only paths whose parent directory is a valid Snapshot id can be
+            # reconstructed into ArchiveResult rows.
+            snapshot = Snapshot.objects.filter(id=plugin_dir.parent.name).first()
+        except ValidationError:
+            continue
+        if snapshot is None:
+            continue
+        result, created = ArchiveResult.objects.get_or_create(
+            snapshot=snapshot,
+            plugin=plugin_dir.name,
+            hook_name=Path(hook_script_name).stem,
+            defaults={
+                "status": ArchiveResult.StatusChoices.QUEUED,
+            },
+        )
+        if result.status == ArchiveResult.StatusChoices.QUEUED:
+            requeue_snapshot = False
+            # A runner can die after the hook Process exits but before the
+            # ProcessCompletedEvent projector links/finalizes ArchiveResult.
+            # Reconstruct only that exact hook row from the durable Process row.
+            output_files, output_size, output_mimetypes = _collect_output_metadata(plugin_dir)
+            result.process = process
+            if _is_signal_interrupted_exit(process.exit_code):
+                # The owning runner died or was asked to stop while the hook was
+                # still active. Keep the work item queued so takeover retries the
+                # same hook; treating an unknown signal exit as success would
+                # silently skip unfinished side effects.
+                result.output_files = {}
+                result.output_size = 0
+                result.output_mimetypes = ""
+                result.output_str = ""
+                result.status = ArchiveResult.StatusChoices.QUEUED
+                requeue_snapshot = True
+            else:
+                result.output_files = output_files
+                result.output_size = output_size
+                result.output_mimetypes = output_mimetypes
+                result.output_str = process.stderr if process.exit_code not in (0, None) else ""
+                result.status = (
+                    ArchiveResult.StatusChoices.FAILED
+                    if process.exit_code not in (0, None)
+                    else (ArchiveResult.StatusChoices.SUCCEEDED if output_files else ArchiveResult.StatusChoices.NORESULTS)
+                )
+            result.save(
+                update_fields=[
+                    "process",
+                    "output_files",
+                    "output_size",
+                    "output_mimetypes",
+                    "output_str",
+                    "status",
+                    "modified_at",
+                ],
+            )
+            if requeue_snapshot:
+                Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
+        if created:
+            cleaned["archiveresults_missing_for_orphaned_hook_processes"] += 1
+            Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
     started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__gt=now),
     )

archivebox/core/routes_util.py

@@ -9,7 +9,7 @@ from archivebox.config.common import get_config
 
 _SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
 _SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P<suffix>[0-9a-fA-F]{12})$")
-_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api", "public")
+_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api")
 
 
 def split_host_port(host: str) -> tuple[str, str | None]:
@@ -102,7 +102,7 @@ def _with_port(host: str, port: str | None) -> str:
 
 
 def strip_role_subdomain(host: str) -> str:
-    """Strip leading ``admin.`` / ``web.`` / ``api.`` / ``public.`` / ``snap-*.``
+    """Strip leading ``admin.`` / ``web.`` / ``api.`` / ``snap-*.``
     labels from a host (preserving the port). Strips repeatedly so an
     already-compounded host like ``snap-X.snap-X.<base>`` reduces all the
     way down to ``<base>``.
@@ -172,13 +172,12 @@ def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kw
             return f"{scheme}://{_with_port('archivebox.localhost', req_port)}"
         # C) Per-request fallback: when ``BASE_URL`` is unset and CSRF didn't
         # give us a single origin, trust the request's Host header — but first
-        # peel off any ``admin.`` / ``web.`` / ``api.`` / ``public.`` /
-        # ``snap-*.`` label. Otherwise the URL builders below prepend their own
-        # role label onto a host that already carries one, producing the
-        # ``snap-X.snap-X.snap-X.<base>`` compounding bug. Django has already
-        # admitted the host via ALLOWED_HOSTS; the misconfig banner surfaces
-        # the case where the resulting URL doesn't match what the operator
-        # probably intended.
+        # peel off any ``admin.`` / ``web.`` / ``api.`` / ``snap-*.`` label.
+        # Otherwise the URL builders below prepend their own role label onto a
+        # host that already carries one, producing the ``snap-X.snap-X.snap-X``
+        # compounding bug. Django has already admitted the host via
+        # ALLOWED_HOSTS; the misconfig banner surfaces the case where the
+        # resulting URL doesn't match what the operator probably intended.
         canonical_host = strip_role_subdomain(request.get_host())
         return f"{scheme}://{canonical_host}"
 
@@ -220,13 +219,6 @@ def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) ->
     return _build_base_host("api", config=config)
 
 
-def get_public_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_host(config=config)
-    return _build_base_host("public", config=config)
-
-
 def get_snapshot_subdomain(snapshot_id: str) -> str:
     normalized = re.sub(r"[^0-9a-fA-F]", "", snapshot_id or "")
     suffix = (normalized[-12:] if len(normalized) >= 12 else normalized).lower()
@@ -330,13 +322,6 @@ def get_api_base_url(request=None, config: dict[str, Any] | None = None, **confi
     return _build_base_url_for_host(_build_base_host("api", request=request, config=config), request=request, config=config)
 
 
-def get_public_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_url(request=request, config=config)
-    return _build_base_url_for_host(_build_base_host("public", request=request, config=config), request=request, config=config)
-
-
 def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     if not config.USES_SUBDOMAIN_ROUTING:

archivebox/core/settings.py

@@ -13,7 +13,7 @@ import archivebox
 
 from archivebox.config.constants import CONSTANTS
 from archivebox.config.common import get_config
-from archivebox.core.routes_util import normalize_base_url, get_admin_base_url, get_api_base_url
+from archivebox.core.routes_util import get_api_base_url, get_admin_base_url, get_base_url, normalize_base_url
 from .settings_logging import SETTINGS_LOGGING
 
 
@@ -72,15 +72,19 @@ INSTALLED_APPS = [
     "archivebox.crawls",  # handles Crawl and CrawlSchedule models and management (depends on core)
     "archivebox.progressmonitor",  # live progress endpoint and admin monitor template
     "archivebox.api",  # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
+    "abx_plugins.plugins.opencode",
     # 3rd-party apps from PyPI that need to be loaded last
     "admin_data_views",  # handles rendering some convenient automatic read-only views of data in Django admin
     "django_extensions",  # provides Django Debug Toolbar (and other non-debug helpers)
 ]
 
+DJANGO_OBJECT_ACTIONS_DEFAULT_HTTP_METHOD = "POST"
+
 
 MIDDLEWARE = [
     "archivebox.core.middleware.TimezoneMiddleware",
     "django.middleware.security.SecurityMiddleware",
+    "archivebox.core.middleware.AdminCookieIsolationMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.common.CommonMiddleware",
     "archivebox.api.middleware.ApiCorsMiddleware",
@@ -416,13 +420,28 @@ SECURE_BROWSER_XSS_FILTER = True
 SECURE_CONTENT_TYPE_NOSNIFF = True
 SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
 
-CSRF_COOKIE_SECURE = False
-SESSION_COOKIE_SECURE = False
+# When BASE_URL is an https:// URL the deployment is HTTPS end-to-end, typically
+# behind a TLS-terminating proxy/tunnel (the bundled traefik/cloudflared profiles,
+# or your own caddy/traefik/nginx) where the proxy -> archivebox hop is plain HTTP, so
+# request.is_secure() / request.scheme would otherwise report http. Honour the
+# proxy's X-Forwarded-Proto so request-derived schemes are correct, and mark the
+# admin session + CSRF cookies Secure so auth cookies are never sent in cleartext.
+# Derived from the RESOLVED base URL's scheme — no separate flag to keep in sync.
+# get_base_url() also covers deployments that only set CSRF_TRUSTED_ORIGINS (the
+# implicit-BASE_URL fallback used on 0.7.x->0.9.x upgrades), so HTTPS hardening
+# isn't lost until BASE_URL is migrated. A plain-http base (e.g. local
+# http://archivebox.localhost:8000) keeps the defaults below.
+BASE_URL_IS_HTTPS = get_base_url(config=CONFIG).strip().lower().startswith("https://")
+if BASE_URL_IS_HTTPS:
+    SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+CSRF_COOKIE_SECURE = BASE_URL_IS_HTTPS
+SESSION_COOKIE_SECURE = BASE_URL_IS_HTTPS
 SESSION_COOKIE_HTTPONLY = True
 SESSION_COOKIE_NAME = f"archivebox_sessionid_{CONSTANTS.COLLECTION_ID}"
 CSRF_COOKIE_NAME = f"archivebox_csrftoken_{CONSTANTS.COLLECTION_ID}"
 # Auth cookies are intentionally scoped to the exact host that set them so
-# the admin session is NOT readable from public.* / web.* / api.* — that
+# the admin session is NOT readable from web.* / api.* — that
 # split is a security boundary, not a UX choice. Subdomains that need to
 # render an "is the user logged in?" indicator must use the single-bit
 # `archivebox_admin_logged_in` hint cookie set by core/middleware.py

archivebox/core/snapshot_status.py

@@ -0,0 +1,25 @@
+__package__ = "archivebox.core"
+
+from django.db.models import QuerySet
+
+
+def snapshot_status_values() -> tuple[str, ...]:
+    from archivebox.core.models import Snapshot
+
+    return tuple(Snapshot.StatusChoices.values)
+
+
+def normalize_snapshot_status(status: str | None) -> str | None:
+    value = str(status or "").strip().lower()
+    if not value:
+        return None
+
+    valid_statuses = snapshot_status_values()
+    if value not in valid_statuses:
+        raise ValueError(f"Invalid snapshot status: {status}. Expected one of: {', '.join(valid_statuses)}")
+    return value
+
+
+def filter_snapshots_by_status(queryset: QuerySet, status: str | None) -> QuerySet:
+    value = normalize_snapshot_status(status)
+    return queryset.filter(status=value) if value else queryset

archivebox/core/tag_util.py

@@ -114,9 +114,9 @@ def add_snapshot_counts(tags: list[Tag], snapshot_queryset: QuerySet[Snapshot] |
     queryset = SnapshotTag.objects.filter(tag_id__in=tag_ids)
     if snapshot_queryset is not None:
         queryset = queryset.filter(snapshot_id__in=snapshot_queryset.values("id"))
-    counts = {row["tag_id"]: row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))}
+    counts = {int(row["tag_id"]): row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))}
     for tag in tags:
-        tag.num_snapshots = counts.get(tag.pk, 0)
+        tag.num_snapshots = counts.get(int(tag.pk), 0)
 
 
 def get_tag_creator_choices() -> list[tuple[str, str]]:

archivebox/core/takeover_util.py

@@ -18,10 +18,18 @@ def runtime_stack_owner_types():
     from archivebox.machine.models import Process
 
     return (
-        Process.TypeChoices.UPDATE,
         Process.TypeChoices.SERVER,
         Process.TypeChoices.ORCHESTRATOR,
+    )
+
+
+def foreground_runner_owner_types():
+    from archivebox.machine.models import Process
+
+    return (
+        Process.TypeChoices.SERVER,
         Process.TypeChoices.ADD,
+        Process.TypeChoices.UPDATE,
     )
 
 
@@ -70,15 +78,13 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
     if exclude_id is not None:
         base_qs = base_qs.exclude(id=exclude_id)
 
-    top_level_types = (
-        Process.TypeChoices.UPDATE,
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ADD,
-    )
     for qs in (
-        base_qs.filter(process_type__in=top_level_types),
+        # Only server parents own HTTP runtime leadership. Foreground add/update
+        # commands can own runner/sonic components, but server startup must never
+        # wait behind them before binding Daphne.
+        base_qs.filter(process_type=Process.TypeChoices.SERVER),
         # A foreground `archivebox run` process is allowed to own the runtime
-        # stack when no server/update/add parent is alive. A runner launched by
+        # stack when no server/add parent is alive. A runner launched by
         # supervisord is only a child worker; after its parent is killed it must
         # not keep stealing leadership from the next foreground command.
         base_qs.filter(process_type=Process.TypeChoices.ORCHESTRATOR).exclude(parent__process_type=Process.TypeChoices.SUPERVISORD),
@@ -95,6 +101,30 @@ def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:
     return bool(owner and owner.id == command.id)
 
 
+def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
+    from archivebox.machine.models import Machine, Process
+
+    machine = Machine.current()
+    qs = Process.objects.filter(
+        machine=machine,
+        status=Process.StatusChoices.RUNNING,
+        pwd=str(data_dir),
+        process_type__in=foreground_runner_owner_types(),
+    )
+    if exclude_id is not None:
+        qs = qs.exclude(id=exclude_id)
+    for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+        if proc.is_running:
+            return proc
+        proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+    return None
+
+
+def command_owns_foreground_runner(command, *, data_dir: str | Path) -> bool:
+    owner = foreground_runner_owner(data_dir=data_dir)
+    return bool(owner and owner.id == command.id)
+
+
 def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
     try:
         from archivebox.config.common import get_config
@@ -108,9 +138,9 @@ def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
     if not names and owner is not None:
         from archivebox.machine.models import Process
 
-        if owner.process_type in {Process.TypeChoices.SERVER, Process.TypeChoices.ADD}:
+        if owner.process_type == Process.TypeChoices.SERVER:
             names = ["orchestrator", "server"]
-        elif owner.process_type in {Process.TypeChoices.UPDATE, Process.TypeChoices.ORCHESTRATOR}:
+        elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
             names = ["orchestrator"]
 
     return ", ".join(dict.fromkeys(names)) or "runtime stack"
@@ -292,3 +322,26 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva
     command.modified_at = timezone.now()
     command.save(update_fields=["modified_at"])
     return {"resumed": announced, "previous_owner_pid": previous_owner_pid}
+
+
+def standby_until_foreground_runner_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> dict[str, object]:
+    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
+
+    announced = False
+    previous_owner_pid = None
+    while not command_owns_foreground_runner(command, data_dir=data_dir):
+        reap_foreground_supervisord_process()
+        if not announced:
+            owner = foreground_runner_owner(data_dir=data_dir)
+            owner_pid = owner.pid if owner else "unknown"
+            previous_owner_pid = owner_pid
+            rprint(
+                f"[yellow][*] A newer archivebox process took over the orchestrator, sonic "
+                f"(pid={owner_pid}). Work will continue there, and will resume here if that process exits and work still remains.[/yellow]",
+                file=sys.stderr,
+            )
+            announced = True
+        time.sleep(interval)
+    command.modified_at = timezone.now()
+    command.save(update_fields=["modified_at"])
+    return {"resumed": announced, "previous_owner_pid": previous_owner_pid}

archivebox/core/templatetags/core_tags.py

@@ -4,6 +4,9 @@ from django import template
 from django.contrib.admin.templatetags.base import InclusionAdminNode
 from django.utils.safestring import mark_safe
 from django.utils.html import escape
+from django.templatetags.static import static
+from django.utils import timezone
+from django.utils.text import Truncator
 
 from pathlib import Path
 
@@ -17,7 +20,6 @@ from archivebox.plugins.discovery import (
 from archivebox.core.routes_util import (
     canonical_base_host_for_request,
     get_admin_base_url,
-    get_public_base_url,
     get_web_base_url,
     get_snapshot_base_url,
     build_snapshot_url,
@@ -495,11 +497,6 @@ def web_base_url(context) -> str:
     return get_web_base_url(request=context.get("request"), config=context.get("CONFIG"))
 
 
-@register.simple_tag(takes_context=True)
-def public_base_url(context) -> str:
-    return get_public_base_url(request=context.get("request"), config=context.get("CONFIG"))
-
-
 @register.simple_tag(takes_context=True)
 def snapshot_base_url(context, snapshot) -> str:
     snapshot_id = _snapshot_id(snapshot)
@@ -512,6 +509,118 @@ def snapshot_url(context, snapshot, path: str = "") -> str:
     return build_snapshot_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
 
 
+@register.simple_tag(takes_context=True)
+def snapshot_index_row(context, link) -> str:
+    snapshot_id = str(_snapshot_id(link))
+    request = context.get("request")
+    config = context.get("CONFIG")
+    snapshot_base = get_snapshot_base_url(snapshot_id, request=request, config=config)
+    screenshot_plugin_url = f"{snapshot_base}/screenshot/screenshot.png"
+    extension_screenshot_1_url = f"{snapshot_base}/chrome_extension_screenshot/screenshot-1.png"
+    extension_screenshot_url = f"{snapshot_base}/chrome_extension_screenshot/screenshot.png"
+    favicon_plugin_url = f"{snapshot_base}/favicon/favicon.ico"
+    favicon_root_url = f"{snapshot_base}/favicon.ico"
+
+    status = getattr(link, "status", None) or "unknown"
+    bookmarked_at = getattr(link, "bookmarked_at", None)
+    timestamp = getattr(link, "timestamp", "")
+    if bookmarked_at:
+        bookmarked_at = timezone.localtime(bookmarked_at)
+        date_text = bookmarked_at.strftime("%Y-%m-%d")
+        time_text = bookmarked_at.strftime("%H:%M")
+        sort_value = str(bookmarked_at.timestamp())
+        title_time = f"Bookmarked: {bookmarked_at} ({timestamp})"
+    else:
+        date_text = ""
+        time_text = ""
+        sort_value = ""
+        title_time = f"Bookmarked: ({timestamp})"
+
+    url = getattr(link, "url", "") or ""
+    title = getattr(link, "title", "") or ""
+    is_pending = status in {"queued", "started", "backoff"}
+    title_text = title or ("Loading..." if is_pending else url)
+    tags_str = link.tags_str() if callable(getattr(link, "tags_str", None)) else getattr(link, "tags_str", "")
+    tag_html = "".join(f'<span class="snapshot-tag">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag)
+    if tag_html:
+        tag_cell = f'<span class="snapshot-tags">{tag_html}</span>'
+    else:
+        tag_cell = '<span class="empty-value">...</span>'
+
+    num_outputs = int(getattr(link, "num_outputs", 0) or 0)
+    icons = link.icons() if callable(getattr(link, "icons", None)) else getattr(link, "icons", "")
+    icons_cell = str(icons) if icons else '<span class="empty-value">...</span>'
+    archive_size = int(getattr(link, "archive_size", 0) or 0)
+    size_cell = file_size(archive_size) if archive_size else '<span class="empty-value">...</span>'
+    output_plural = "" if num_outputs == 1 else "s"
+
+    if is_pending:
+        preview_html = (
+            '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress">'
+            f'<img src="{escape(static("spinner.gif"))}" alt="" decoding="async" loading="lazy">'
+            "</span>"
+        )
+    else:
+        preview_html = (
+            f'<img src="{escape(screenshot_plugin_url)}" '
+            f'data-fallbacks="{escape(extension_screenshot_1_url)},{escape(extension_screenshot_url)}" '
+            'onerror="nextPublicSnapshotPreview(this)" class="snapshot-preview screenshot" alt="" decoding="async" loading="lazy">'
+        )
+
+    html = f"""
+<tr class="snapshot-row status-{escape(status)}">
+    <td class="snapshot-time" title="{escape(title_time)}" data-sort="{escape(sort_value)}">
+        <a href="{escape(snapshot_base)}/index.html">
+            <span>{escape(date_text)}</span>
+            <small>{escape(time_text)}</small>
+        </a>
+    </td>
+    <td class="snapshot-preview-cell">
+        <a href="{escape(snapshot_base)}/index.html" title="Open archived snapshot">
+            {preview_html}
+        </a>
+    </td>
+    <td class="snapshot-title-cell" title="{escape(title or url)}">
+        <div class="snapshot-title-line">
+            <a href="{escape(snapshot_base)}/index.html" class="snapshot-favicon-link" title="Open archived snapshot">
+                <img src="{escape(favicon_plugin_url)}"
+                     data-fallbacks="{escape(favicon_root_url)}"
+                     onerror="nextPublicSnapshotPreview(this)"
+                     class="link-favicon"
+                     alt=""
+                     decoding="async"
+                     loading="lazy">
+            </a>
+            <a href="{escape(snapshot_base)}/index.html" class="snapshot-title">
+                {escape(Truncator(title_text).chars(110))}
+            </a>
+        </div>
+        <a href="{escape(url)}" class="snapshot-url" title="{escape(url)}" target="_blank" rel="noopener noreferrer">
+            {escape(url)}
+        </a>
+    </td>
+    <td class="snapshot-tags-cell">
+        {tag_cell}
+    </td>
+    <td class="snapshot-status-cell">
+        <span class="snapshot-status">{escape(status)}</span>
+    </td>
+    <td class="snapshot-files-cell">
+        <span data-number-for="{escape(url)}" title="{num_outputs} successful outputs">
+            {icons_cell}
+        </span>
+    </td>
+    <td class="snapshot-size-cell">
+        <a href="{escape(snapshot_base)}/?files=1" title="View archived files">
+            {size_cell}
+        </a>
+        <small>{num_outputs} output{output_plural}</small>
+    </td>
+</tr>
+"""
+    return mark_safe(html)
+
+
 @register.simple_tag(takes_context=True)
 def snapshot_preview_url(context, snapshot, path: str = "") -> str:
     snapshot_id = _snapshot_id(snapshot)

archivebox/core/urls.py

@@ -19,6 +19,7 @@ from archivebox.core.views import (
     SnapshotView,
     SnapshotPathView,
     SnapshotReplayView,
+    SnapshotReplayAuthView,
     OriginalDomainReplayView,
     PublicIndexView,
     AddView,
@@ -26,6 +27,8 @@ from archivebox.core.views import (
     HealthCheckView,
 )
 from archivebox.progressmonitor.views import live_progress_view
+from archivebox.search.views import public_snapshot_search_stream_view
+from abx_plugins.plugins.opencode.views import opencode_proxy_view
 
 
 # GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
@@ -41,6 +44,9 @@ urlpatterns = [
     path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}),
     path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}),
     path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"),
+    re_path(r"^admin/agent/?(?=$|opencode)", include("abx_plugins.plugins.opencode.urls")),
+    re_path(r"^(?P<path>assets/.*)$", opencode_proxy_view, name="opencode-assets"),
+    path("public/search-stream/", public_snapshot_search_stream_view, name="public-search-stream"),
     path("public/", PublicIndexView.as_view(), name="public-index"),
     path("public.html", RedirectView.as_view(url="/public/"), name="public-index-html"),
     path("archive/", RedirectView.as_view(url="/")),
@@ -65,6 +71,7 @@ urlpatterns = [
         name="snapshot-path-nodate",
     ),
     path("admin/core/snapshot/add/", RedirectView.as_view(url="/add/")),
+    path("admin/core/snapshot/replay-auth/", SnapshotReplayAuthView.as_view(), name="snapshot-replay-auth"),
     path("add/", AddView.as_view(), name="add"),
     # ``query_string=True`` preserves the ``?next=…`` param that Django's
     # auth/login mixins append, so e.g. ``UserPassesTestMixin`` redirecting

archivebox/core/views.py

@@ -16,9 +16,14 @@ from django.utils.safestring import mark_safe
 from django.views import View
 from django.views.generic.list import ListView
 from django.views.generic import FormView
-from django.db.models import Q
+from django.db.models import Case, IntegerField, Q, Value, When
+from django.core.paginator import InvalidPage
 from django.contrib import messages
+from django.conf import settings
+from django.contrib.auth import HASH_SESSION_KEY, SESSION_KEY, get_user_model
 from django.contrib.auth.mixins import UserPassesTestMixin
+from django.contrib.sessions.models import Session
+from django.core import signing
 from django.views.decorators.csrf import csrf_exempt
 from django.utils.decorators import method_decorator
 
@@ -37,10 +42,12 @@ from archivebox.config.common import (
     get_config,
     get_all_configs,
     get_request_config,
+    _plugin_config_properties,
     redact_sensitive_config,
 )
+from archivebox.config.common import PLUGIN_CONFIG_SCHEMAS
 from archivebox.config.configset import BaseConfigSet
-from archivebox.misc.paginators import CountlessPaginator
+from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.misc.util import (
     base_url,
     filter_queryset_by_uuid_substring,
@@ -57,11 +64,13 @@ from archivebox.search.config import (
     get_search_mode_base,
     get_search_mode_options,
 )
-from archivebox.search.query import apply_snapshot_search
+from archivebox.search.views import get_cached_public_search_state
 
-from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
 from archivebox.core.permissions import (
+    PERMISSIONS_PRIVATE,
     PERMISSIONS_PUBLIC,
+    PERMISSIONS_UNLISTED,
     can_view_snapshot,
     direct_snapshots_queryset,
     filter_personas_by_permissions,
@@ -82,7 +91,6 @@ from archivebox.core.routes_util import (
 from archivebox.core.forms import AddLinkForm
 from archivebox.plugins.forms import get_plugin_config_binary_urls
 from archivebox.crawls.models import Crawl
-from archivebox.workers.models import RETRY_AT_MAX
 from archivebox.plugins.discovery import discover_plugin_configs
 from archivebox.plugins.views import get_config_definition_link
 from archivebox.progressmonitor.views import live_progress_view, progress_endpoint
@@ -125,6 +133,125 @@ def _admin_login_redirect_or_forbidden(request: HttpRequest):
     return HttpResponseForbidden("ArchiveBox is running with the control plane disabled in this security mode.")
 
 
+REPLAY_AUTH_SALT = "archivebox.private-snapshot-replay"
+REPLAY_COOKIE_PREFIX = f"archivebox_replay_{CONSTANTS.COLLECTION_ID}_"
+REPLAY_GRANT_MAX_AGE = 60
+
+
+def _replay_cookie_name(snapshot: Snapshot) -> str:
+    return f"{REPLAY_COOKIE_PREFIX}{str(snapshot.id).replace('-', '')[-12:]}"
+
+
+def _clean_replay_next(path: str | None) -> str:
+    """Only allow same-snap relative replay paths; grants must never redirect off-host."""
+    path = f"/{(path or 'index.html').lstrip('/')}"
+    parsed = urlparse(path)
+    if parsed.scheme or parsed.netloc or path.startswith("//"):
+        return "/index.html"
+    return path
+
+
+def _replay_payload_is_valid(payload: dict, snapshot: Snapshot) -> bool:
+    """A replay cookie is not its own auth source; it must point at a live admin session.
+
+    Replayed pages can execute hostile JS, so admin cookies stay host-only on admin.*.
+    The snap host gets only this host-only HttpOnly cookie, and every request checks
+    that the original Django session still exists and still belongs to an active staff user.
+    Logout, session expiry, user deletion/deactivation, or password auth-hash rotation all
+    make the replay cookie inert without needing admin.* to delete a cookie on snap-*.
+    """
+    if payload.get("snapshot_id") != str(snapshot.id):
+        return False
+    try:
+        session = Session.objects.get(session_key=str(payload.get("session_key") or ""))
+        session_data = session.get_decoded()
+        user_id = str(session_data.get(SESSION_KEY) or "")
+        auth_hash = str(session_data.get(HASH_SESSION_KEY) or "")
+        user = get_user_model().objects.get(pk=user_id)
+    except Exception:
+        return False
+    return (
+        str(payload.get("user_id")) == user_id
+        and str(payload.get("auth_hash") or "") == auth_hash
+        and user.is_active
+        and user.is_staff
+        and auth_hash == user.get_session_auth_hash()
+    )
+
+
+def _has_replay_cookie(request: HttpRequest, snapshot: Snapshot) -> bool:
+    value = request.COOKIES.get(_replay_cookie_name(snapshot))
+    if not value:
+        return False
+    try:
+        payload = signing.loads(value, salt=REPLAY_AUTH_SALT, max_age=settings.SESSION_COOKIE_AGE)
+    except signing.BadSignature:
+        return False
+    return isinstance(payload, dict) and _replay_payload_is_valid(payload, snapshot)
+
+
+def _private_snapshot_auth_redirect(request: HttpRequest, snapshot: Snapshot, path: str = "", *, preserve_query: bool = True):
+    next_path = _clean_replay_next(path or "index.html")
+    if preserve_query and request.META.get("QUERY_STRING"):
+        next_path = f"{next_path}?{request.META['QUERY_STRING']}"
+    target = build_admin_url(
+        f"/admin/core/snapshot/replay-auth/?snapshot={snapshot.id}&next={quote(next_path, safe='')}",
+        request=request,
+    )
+    return redirect(target)
+
+
+def _replay_auth_response(request: HttpRequest, snapshot: Snapshot):
+    try:
+        payload = signing.loads(str(request.GET.get("grant") or ""), salt=REPLAY_AUTH_SALT, max_age=REPLAY_GRANT_MAX_AGE)
+    except signing.BadSignature:
+        return _private_snapshot_auth_redirect(request, snapshot, "index.html", preserve_query=False)
+
+    if not isinstance(payload, dict) or not _replay_payload_is_valid(payload, snapshot):
+        return _private_snapshot_auth_redirect(request, snapshot, "index.html", preserve_query=False)
+
+    cookie_value = signing.dumps(payload, salt=REPLAY_AUTH_SALT)
+    response = redirect(_clean_replay_next(request.GET.get("next")))
+    response.set_cookie(
+        _replay_cookie_name(snapshot),
+        cookie_value,
+        max_age=settings.SESSION_COOKIE_AGE,
+        secure=request.is_secure(),
+        httponly=True,
+        samesite="Lax",
+    )
+    return response
+
+
+class SnapshotReplayAuthView(View):
+    """Admin-only handoff that lets a snap host mint its own replay cookie.
+
+    admin.* cannot set a host-only cookie for snap-* (browsers forbid that), and
+    widening the real Django session cookie to *.archivebox.localhost would let XSS
+    in replayed pages hit the admin UI. Instead admin.* proves the user is logged in
+    with a short URL grant, then snap-* validates it and sets a snap-host-only cookie.
+    """
+
+    def get(self, request: HttpRequest):
+        if not is_admin_user(request):
+            return redirect(f"{build_admin_url('/admin/login/', request=request)}?next={quote(request.get_full_path(), safe='')}")
+
+        snapshot = _find_snapshot_by_ref(str(request.GET.get("snapshot") or ""))
+        if not snapshot:
+            raise Http404
+
+        payload = {
+            "snapshot_id": str(snapshot.id),
+            "user_id": str(request.user.pk),
+            "session_key": request.session.session_key,
+            "auth_hash": request.user.get_session_auth_hash(),
+        }
+        grant = signing.dumps(payload, salt=REPLAY_AUTH_SALT)
+        next_path = _clean_replay_next(request.GET.get("next"))
+        target = build_snapshot_url(str(snapshot.id), "_auth", request=request, config=get_request_config(request))
+        return redirect(f"{target}?grant={quote(grant, safe='')}&next={quote(next_path, safe='')}")
+
+
 class HomepageView(View):
     def get(self, request):
         request_config = get_request_config(request)
@@ -204,7 +331,7 @@ class SnapshotView(View):
         hidden_card_plugins = {"archivedotorg", "favicon", "title"}
         outputs = [
             out
-            for out in snapshot.discover_outputs(include_filesystem_fallback=False)
+            for out in snapshot.discover_outputs(include_filesystem_fallback=True)
             if (out.get("size") or 0) > 0 and out.get("name") not in hidden_card_plugins
         ]
         archiveresults = {out["name"]: out for out in outputs}
@@ -362,7 +489,7 @@ class SnapshotView(View):
                 try:
                     snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
                     if not can_view_snapshot(request, snapshot):
-                        return _admin_login_redirect_or_forbidden(request)
+                        return _private_snapshot_auth_redirect(request, snapshot, archivefile or "index.html")
                     canonical_base = snapshot.url_path
                     if canonical_base != snapshot.legacy_archive_path:
                         target_path = f"/{canonical_base}/{archivefile or 'index.html'}"
@@ -571,7 +698,7 @@ class SnapshotPathView(View):
         if snapshot_id:
             snapshot = _find_snapshot_by_ref(snapshot_id)
             if snapshot and not can_view_snapshot(request, snapshot):
-                return _admin_login_redirect_or_forbidden(request)
+                return _private_snapshot_auth_redirect(request, snapshot, path or "index.html")
         else:
             # fuzzy lookup by date + domain/url (most recent)
             username_lookup = "system" if username == "web" else username
@@ -730,7 +857,7 @@ def _replay_path_visible(request: HttpRequest, path: Path) -> bool:
     if not snapshot_id:
         return False
     snapshot = Snapshot.objects.filter(id=snapshot_id).select_related("crawl", "crawl__created_by").first()
-    if not snapshot or not can_view_snapshot(request, snapshot):
+    if not snapshot or (not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot)):
         return False
     request.archivebox_config = get_request_config(request, resolve_plugins=False)
     return True
@@ -953,8 +1080,10 @@ class SnapshotHostView(View):
 
         if not snapshot:
             raise Http404
-        if not can_view_snapshot(request, snapshot):
-            return _admin_login_redirect_or_forbidden(request)
+        if path == "_auth":
+            return _replay_auth_response(request, snapshot)
+        if not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot):
+            return _private_snapshot_auth_redirect(request, snapshot, path)
 
         canonical_host = get_snapshot_host(str(snapshot.id), config=request_config)
         if not host_matches(request.get_host(), canonical_host):
@@ -973,8 +1102,10 @@ class SnapshotReplayView(View):
         snapshot = _find_snapshot_by_ref(snapshot_id)
         if not snapshot:
             raise Http404
-        if not can_view_snapshot(request, snapshot):
-            return _admin_login_redirect_or_forbidden(request)
+        if path == "_auth":
+            return _replay_auth_response(request, snapshot)
+        if not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot):
+            return _private_snapshot_auth_redirect(request, snapshot, path)
 
         return _serve_snapshot_replay(request, snapshot, path)
 
@@ -997,7 +1128,8 @@ class PublicIndexView(ListView):
     template_name = "public_index.html"
     model = Snapshot
     ordering = ["-bookmarked_at", "-created_at"]
-    paginator_class = CountlessPaginator
+    paginator_class = AcceleratedPaginator
+    public_page_scan_chunk_size = 50
 
     def get_paginate_by(self, queryset):
         runtime_config = self.__dict__.get("runtime_config")
@@ -1005,12 +1137,66 @@ class PublicIndexView(ListView):
             self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
         return runtime_config.SNAPSHOTS_PER_PAGE
 
+    def _base_public_snapshot_fields(self) -> tuple[str, ...]:
+        return (
+            "id",
+            "created_at",
+            "modified_at",
+            "url",
+            "timestamp",
+            "bookmarked_at",
+            "title",
+            "downloaded_at",
+            "status",
+            "output_size",
+            "permissions",
+        )
+
+    def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot] | None:
+        target_count = page_number * page_size
+        public_snapshots: list[Snapshot] = []
+        scanned = 0
+        chunk_size = max(self.public_page_scan_chunk_size, page_size)
+        ordered_snapshots = Snapshot.objects.order_by(*self.ordering).only(*self._base_public_snapshot_fields())
+
+        while len(public_snapshots) < target_count:
+            chunk = list(ordered_snapshots[scanned : scanned + chunk_size])
+            if not chunk:
+                break
+            scanned += len(chunk)
+            public_snapshots.extend(snapshot for snapshot in chunk if snapshot.permissions == PERMISSIONS_PUBLIC)
+
+        start = (page_number - 1) * page_size
+        return public_snapshots[start:target_count]
+
+    def paginate_queryset(self, queryset, page_size):
+        if self.request.GET.get("q", default="").strip():
+            return super().paginate_queryset(queryset, page_size)
+
+        public_count = self.get_exact_public_snapshot_count()
+        paginator = self.get_paginator(range(public_count), page_size)
+        page_kwarg = self.kwargs.get(self.page_kwarg)
+        page_query = self.request.GET.get(self.page_kwarg)
+        page_number = page_kwarg or page_query or 1
+
+        try:
+            page = paginator.page(page_number)
+        except InvalidPage as err:
+            raise Http404(f"Invalid page ({page_number}): {err}") from err
+
+        object_list = self._ordered_public_page_from_order_index(page_number=page.number, page_size=page_size)
+        page.object_list = object_list
+        return paginator, page, object_list, page.has_other_pages()
+
     def get_context_data(self, **kwargs):
         runtime_config = self.__dict__.get("runtime_config")
         if runtime_config is None:
             self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
         search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
         search_mode_backend = get_search_mode_backend(search_mode, config=runtime_config)
+        query = self.request.GET.get("q", default="").strip()
+        public_search_state = self.__dict__.get("public_search_state")
+        public_search_pending = bool(query and (public_search_state is None or not public_search_state.get("done")))
         context = {
             **super().get_context_data(**kwargs),
             "VERSION": VERSION,
@@ -1020,15 +1206,18 @@ class PublicIndexView(ListView):
             "WEB_BASE_URL": build_web_url(request=self.request, config=runtime_config),
             "search_mode": search_mode,
             "search_mode_options": get_search_mode_options(config=runtime_config),
+            "public_search_stream_pending": public_search_pending,
         }
         context["show_search_index_hint"] = bool(
-            self.request.GET.get("q")
+            query
+            and not public_search_pending
             and get_search_mode_base(search_mode, config=runtime_config) == "deep"
             and search_mode_backend
             and context["paginator"].count == 0,
         )
         snapshots = list(context.get("object_list") or ())
         icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots}
+        tag_names_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
         progress_by_snapshot: dict[str, dict[str, int]] = {
             str(snapshot.id): {
                 "total": 0,
@@ -1041,6 +1230,14 @@ class PublicIndexView(ListView):
             for snapshot in snapshots
         }
         if icons_by_snapshot:
+            for snapshot_id, tag_name in (
+                SnapshotTag.objects.filter(snapshot_id__in=icons_by_snapshot.keys())
+                .order_by("tag__name")
+                .values_list("snapshot_id", "tag__name")
+                .iterator(chunk_size=1000)
+            ):
+                tag_names_by_snapshot[str(snapshot_id)].append(tag_name)
+
             for snapshot_id, plugin, status in (
                 ArchiveResult.objects.filter(
                     snapshot_id__in=icons_by_snapshot.keys(),
@@ -1068,41 +1265,37 @@ class PublicIndexView(ListView):
             snapshot._icons_compact = True
             snapshot._icons_archive_results = icons_by_snapshot.get(str(snapshot.id), set())
             snapshot._icons_progress_stats = progress_by_snapshot.get(str(snapshot.id), {})
+            snapshot.num_outputs_cached = snapshot._icons_progress_stats.get("succeeded", 0)
+            snapshot._tags_str_cached = ",".join(tag_names_by_snapshot.get(str(snapshot.id), []))
             snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
         context["object_list"] = snapshots
         return context
 
+    def get_exact_public_snapshot_count(self) -> int:
+        hidden_count = Snapshot.objects.filter(permissions=PERMISSIONS_PRIVATE).count()
+        hidden_count += Snapshot.objects.filter(permissions=PERMISSIONS_UNLISTED).count()
+        return Snapshot.objects.count() - hidden_count
+
     def get_queryset(self, **kwargs):
-        qs = (
-            public_snapshots_queryset(super().get_queryset(**kwargs))
-            .select_related("crawl__created_by")
-            .annotate(
-                num_outputs_cached=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
-            )
-            .prefetch_related(
-                "tags",
-            )
-        )
+        qs = public_snapshots_queryset(super().get_queryset(**kwargs)).only(*self._base_public_snapshot_fields())
         query = self.request.GET.get("q", default="").strip()
 
         if not query:
             return qs
 
-        runtime_config = self.__dict__.get("runtime_config")
-        search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
-        try:
-            return apply_snapshot_search(
-                qs,
-                query,
-                search_mode=search_mode,
-                config=runtime_config,
-                ordering=self.ordering,
-            )
-        except Exception as err:
-            print(f"[!] Error while using search backend: {err.__class__.__name__} {err}")
-            if get_search_mode_backend(search_mode, config=runtime_config):
+        cached_state = get_cached_public_search_state(self.request)
+        self.public_search_state = cached_state
+        if cached_state is not None:
+            cached_ids = cached_state.get("ids") or []
+            if not cached_ids:
                 return qs.none()
-            return apply_snapshot_search(qs, query, search_mode="meta", config=runtime_config)
+            search_rank = Case(
+                *(When(pk=snapshot_id, then=Value(index)) for index, snapshot_id in enumerate(cached_ids)),
+                output_field=IntegerField(),
+            )
+            return qs.filter(pk__in=cached_ids).annotate(search_rank=search_rank).order_by("search_rank", *self.ordering)
+
+        return qs.none()
 
     def get(self, *args, **kwargs):
         if self.request.user.is_authenticated:
@@ -1212,6 +1405,8 @@ class AddView(UserPassesTestMixin, FormView):
         }
 
     def _create_crawl_from_form(self, form, *, created_by_id=None) -> Crawl:
+        from archivebox.cli.archivebox_add import add
+
         urls = form.cleaned_data["url"]
         print(f"[+] Adding URL: {urls}")
 
@@ -1244,8 +1439,6 @@ class AddView(UserPassesTestMixin, FormView):
         if persona:
             persona.ensure_dirs()
 
-        from archivebox.config.permissions import HOSTNAME
-
         if created_by_id is None:
             if self.request.user.is_authenticated:
                 created_by_id = self.request.user.pk
@@ -1254,60 +1447,41 @@ class AddView(UserPassesTestMixin, FormView):
 
                 created_by_id = get_or_create_system_user_pk()
 
-        created_by_name = self.request.user.username if self.request.user.is_authenticated else "web"
-
-        # 1. save the provided urls to sources/2024-11-05__23-59-59__web_ui_add_by_user_<user_pk>.txt
-        sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__web_ui_add_by_user_{created_by_id}.txt"
-        sources_file.parent.mkdir(parents=True, exist_ok=True)
-        sources_file.write_text(urls if isinstance(urls, str) else "\n".join(urls))
-
-        # 2. create a new Crawl with the URLs from the file
-        timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
-        urls_content = sources_file.read_text()
-        # Store explicit crawl-scoped overrides; Crawl.save() freezes them
-        # over the resolved persona/user/machine defaults at creation time.
         config = {}
-        if plugins:
-            config["PLUGINS"] = plugins
         effective_config = get_config(persona=persona) if persona else get_config()
-        if crawl_max_concurrent_snapshots != int(effective_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS):
-            config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
         if delete_after != str(effective_config.DELETE_AFTER):
             config["DELETE_AFTER"] = delete_after
-        config["PERMISSIONS"] = permissions
-        if max_urls:
-            config["CRAWL_MAX_URLS"] = max_urls
-        if crawl_max_size:
-            config["CRAWL_MAX_SIZE"] = crawl_max_size
-        if crawl_timeout:
-            config["CRAWL_TIMEOUT"] = crawl_timeout
         if timeout is not None and int(timeout) != int(effective_config.TIMEOUT):
             config["TIMEOUT"] = int(timeout)
-        if snapshot_max_size:
-            config["SNAPSHOT_MAX_SIZE"] = snapshot_max_size
 
-        # Merge custom config overrides
         config.update(plugin_config)
         config.update(custom_config)
         if bool(url_filters.get("only_new")) != bool(effective_config.ONLY_NEW):
             config["ONLY_NEW"] = bool(url_filters.get("only_new"))
-        if url_filters.get("allowlist"):
-            config["URL_ALLOWLIST"] = url_filters["allowlist"]
-        if url_filters.get("denylist"):
-            config["URL_DENYLIST"] = url_filters["denylist"]
-
-        crawl = Crawl.create_scheduler_row(
-            urls=urls_content,
-            max_depth=depth,
-            tags_str=tag,
-            notes=notes,
-            label=f"{created_by_name}@{HOSTNAME}{self.request.path} {timestamp}",
+        crawl, _snapshots = add(
+            urls=urls,
+            depth=depth,
+            max_urls=max_urls,
+            crawl_max_size=crawl_max_size,
+            crawl_timeout=crawl_timeout,
+            snapshot_max_size=snapshot_max_size,
+            crawl_max_concurrent_snapshots=crawl_max_concurrent_snapshots,
+            tag=tag,
+            url_allowlist=url_filters.get("allowlist") or "",
+            url_denylist=url_filters.get("denylist") or "",
+            plugins=plugins,
+            persona=persona.name if persona else "Default",
+            bg=True,
             created_by_id=created_by_id,
             config=config,
-            persona_id=persona.id if persona else None,
-            status=Crawl.StatusChoices.PAUSED if start_paused else Crawl.StatusChoices.QUEUED,
-            retry_at=RETRY_AT_MAX if start_paused else timezone.now(),
         )
+        if notes:
+            crawl.safe_update({"notes": notes}, refresh=False)
+        if permissions and crawl.config.get("PERMISSIONS") != permissions:
+            next_config = {**crawl.config, "PERMISSIONS": permissions}
+            crawl.safe_update({"config": next_config}, refresh=True)
+        if start_paused:
+            crawl.pause()
 
         # 3. create a CrawlSchedule if schedule is provided
         if schedule:
@@ -1325,11 +1499,6 @@ class AddView(UserPassesTestMixin, FormView):
             crawl.schedule = crawl_schedule
             crawl.safe_update({"schedule": crawl_schedule}, refresh=False)
 
-        if not start_paused:
-            from archivebox.services.runner import ensure_background_runner
-
-            ensure_background_runner()
-
         return crawl
 
     def form_valid(self, form):
@@ -1535,15 +1704,10 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
     sources_info = []
 
     # Machine config
-    machine = None
-    machine_admin_url = None
-    try:
-        machine = Machine.current()
-        machine_admin_url = f"/admin/machine/machine/{machine.id}/change/"
-        if machine.config and key in machine.config:
-            sources_info.append(("Machine", redact_sensitive_config(machine.config).get(key), "purple"))
-    except Exception:
-        pass
+    machine = Machine.current()
+    machine_admin_url = machine.admin_change_url
+    if machine.config and key in machine.config:
+        sources_info.append(("Machine", redact_sensitive_config(machine.config).get(key), "purple"))
 
     # Environment variable
     if key in os.environ:
@@ -1557,14 +1721,14 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
 
     # Default value
     default_val = find_config_default(key)
-    if default_val:
+    if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
+        sources_info.append(("Plugin Default", default_val, "gray"))
+    elif default_val:
         sources_info.append(("Default", default_val, "gray"))
 
     # Final computed value
     config_source = find_config_source(key, merged_config)
     final_value = merged_config.get(key, CONFIGS.get(key, None))
-    if config_source == "Environment":
-        final_value = get_config(include_machine=False, redact_sensitive=True).model_dump(mode="json").get(key, CONFIGS.get(key, None))
     is_redacted = final_value == SENSITIVE_CONFIG_VALUE_REDACTED
 
     # Build sources display
@@ -1634,11 +1798,12 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
                 <br/><br/>
                 Priority order (highest to lowest):
                 <ol>
-                    <li><b style="color: blue">Environment</b> - Environment variables</li>
                     <li><b style="color: purple">Machine</b> - Machine-specific overrides
                         {f'<br/><a href="{machine_admin_url}">→ Edit <code>{key}</code> in Machine.config for this server</a>' if machine_admin_url else ""}
                     </li>
-                    <li><b style="color: green">Config File</b> - data/ArchiveBox.conf</li>
+                    <li><b style="color: blue">Environment</b> - process defaults from environment variables</li>
+                    <li><b style="color: green">File</b> - data/ArchiveBox.conf</li>
+                    <li><b style="color: gray">Plugin Default</b> - Default value from plugin config.json</li>
                     <li><b style="color: gray">Default</b> - Default value from code</li>
                 </ol>
                 {f'<br/><b>Tip:</b> To override <code>{key}</code> on this machine, <a href="{machine_admin_url}">edit the Machine.config field</a> and add:<br/><code>{{"\\"{key}\\": "your_value_here"}}</code>' if machine_admin_url and key not in CONSTANTS_CONFIG else ""}

archivebox/crawls/admin.py

@@ -8,13 +8,14 @@ from django import forms
 from django.core.paginator import Paginator
 from django.http import JsonResponse, HttpRequest, HttpResponseBadRequest, HttpResponseNotAllowed
 from django.shortcuts import get_object_or_404, redirect
+from django.template.response import TemplateResponse
 from django.template.loader import render_to_string
 from django.urls import path, reverse
 from django.utils.html import escape, format_html, format_html_join
 from django.utils import timezone
 from django.utils.safestring import mark_safe
 from django.contrib import admin, messages
-from django.db.models import Case, CharField, Count, Q, Value, When
+from django.db.models import Count, F, Q
 
 
 from django_object_actions import action
@@ -79,16 +80,7 @@ def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50,
         failed_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.FAILED),
         started_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.STARTED),
         skipped_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SKIPPED),
-        snapshot_permissions=Case(
-            When(permissions=PERMISSIONS_PUBLIC, then=Value(PERMISSIONS_PUBLIC)),
-            When(permissions=PERMISSIONS_UNLISTED, then=Value(PERMISSIONS_UNLISTED)),
-            When(permissions=PERMISSIONS_PRIVATE, then=Value(PERMISSIONS_PRIVATE)),
-            When(crawl__permissions=PERMISSIONS_PUBLIC, then=Value(PERMISSIONS_PUBLIC)),
-            When(crawl__permissions=PERMISSIONS_UNLISTED, then=Value(PERMISSIONS_UNLISTED)),
-            When(crawl__permissions=PERMISSIONS_PRIVATE, then=Value(PERMISSIONS_PRIVATE)),
-            default=Value(PERMISSIONS_PRIVATE),
-            output_field=CharField(),
-        ),
+        snapshot_permissions=F("permissions"),
     )
 
     page_number = request.GET.get(page_param, 1) if request is not None else 1
@@ -737,6 +729,8 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         self.crawl_admin_base_config = request.archivebox_config
         self.stop_reason_cache = {}
         response = super().changelist_view(request, extra_context)
+        if not isinstance(response, TemplateResponse):
+            return response
         cl = response.context_data.get("cl")
         if cl is not None and not self.should_annotate_snapshot_counts(request):
             self.hydrate_visible_snapshot_counts(cl.result_list)
@@ -925,11 +919,14 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         now = timezone.now()
         updated = 0
         batch = []
-        for crawl in queryset.only("id", "config").iterator(chunk_size=500):
+        crawls_to_update = []
+        for crawl in queryset.only("id", "config", "permissions").iterator(chunk_size=500):
+            old_permissions = crawl.permissions
             config = dict(crawl.config or {})
             config["PERMISSIONS"] = permissions
             crawl.config = config
             crawl.modified_at = now
+            crawls_to_update.append((crawl, old_permissions))
             batch.append(crawl)
             if len(batch) >= 500:
                 Crawl.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
@@ -938,9 +935,11 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         if batch:
             Crawl.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
             updated += len(batch)
+        for crawl, old_permissions in crawls_to_update:
+            crawl.update_child_snapshot_permissions(old_permissions, permissions)
         return updated
 
-    @action(label="Recrawl", description="Create a new crawl with the same settings")
+    @action(label="Recrawl", description="Create a new crawl with the same settings", methods=("POST",))
     def recrawl(self, request, obj):
         """Duplicate this crawl as a new crawl with the same URLs and settings."""
 

archivebox/crawls/models.py

@@ -35,7 +35,7 @@ from archivebox.base_models.models import (
 )
 from archivebox.workers.models import RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine
 from archivebox.crawls.schedule_util import next_run_for_schedule, validate_schedule
-from archivebox.misc.util import validate_url, validate_url_length
+from archivebox.misc.util import parse_date, validate_url, validate_url_length
 
 if TYPE_CHECKING:
     from archivebox.core.models import Snapshot
@@ -311,6 +311,9 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
 
         super().save(*args, **kwargs)
+        old_permissions = getattr(old_crawl, "permissions", None)
+        if old_crawl is not None and old_permissions != self.permissions:
+            transaction.on_commit(lambda: self.update_child_snapshot_permissions(old_permissions, self.permissions))
         if sync_tags:
             next_tag_names = set(self.parse_tag_names(self.tags_str or ""))
             added_tag_names = next_tag_names - previous_tag_names
@@ -340,6 +343,29 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         #         },
         #     )
 
+    def update_child_snapshot_permissions(self, old_permissions: str | None, new_permissions: str | None) -> int:
+        from archivebox.core.models import Snapshot
+
+        normalized_new_permissions = normalize_permissions(new_permissions)
+        now = timezone.now()
+        batch = []
+        updated = 0
+        queryset = self.snapshot_set.filter(Q(permissions=old_permissions) | Q(permissions__isnull=True)).only("id", "config")
+        for snapshot in queryset.iterator(chunk_size=500):
+            config = dict(snapshot.config or {})
+            config["PERMISSIONS"] = normalized_new_permissions
+            snapshot.config = config
+            snapshot.modified_at = now
+            batch.append(snapshot)
+            if len(batch) >= 500:
+                Snapshot.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
+                updated += len(batch)
+                batch.clear()
+        if batch:
+            Snapshot.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
+            updated += len(batch)
+        return updated
+
     @property
     def api_url(self) -> str:
         return str(reverse_lazy("api-1:get_crawl", args=[self.id]))
@@ -495,6 +521,18 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             return []
         return [url.strip() for url in self.urls.split("\n") if url.strip() and not url.strip().startswith("#")]
 
+    def has_internal_input_root(self) -> bool:
+        """Return True when Crawl.urls is preserved source text, not the work queue.
+
+        `archivebox add` creates a synthetic root snapshot to run parser hooks
+        through the same Snapshot lifecycle as every other extractor. In that
+        mode the raw submitted import text must remain in Crawl.urls forever;
+        parsed URLs live as child Snapshot rows and should not be appended back.
+        """
+        from archivebox.core.models import Snapshot
+
+        return self.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL, depth=0).exists()
+
     @staticmethod
     def normalize_domain(value: str) -> str:
         candidate = (value or "").strip().lower()
@@ -562,22 +600,29 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         except re.error:
             return False
 
+    def get_current_config(self, *, refresh: bool = False) -> dict[str, Any]:
+        if refresh and self.pk:
+            config = type(self).objects.filter(pk=self.pk).values_list("config", flat=True).first()
+            if config is not None:
+                self.config = config
+        return dict(self.config or {})
+
     def get_url_allowlist(self, *, use_effective_config: bool = False, snapshot=None) -> list[str]:
         if use_effective_config:
-            from archivebox.config.common import get_config
-
-            config = get_config(crawl=self, snapshot=snapshot)
+            config = self.get_current_config(refresh=True)
         else:
-            config = self.config or {}
+            config = self.get_current_config()
+        if snapshot is not None and snapshot.config:
+            config.update(snapshot.config)
         return self.split_filter_patterns(config.get("URL_ALLOWLIST", ""))
 
     def get_url_denylist(self, *, use_effective_config: bool = False, snapshot=None) -> list[str]:
         if use_effective_config:
-            from archivebox.config.common import get_config
-
-            config = get_config(crawl=self, snapshot=snapshot)
+            config = self.get_current_config(refresh=True)
         else:
-            config = self.config or {}
+            config = self.get_current_config()
+        if snapshot is not None and snapshot.config:
+            config.update(snapshot.config)
         return self.split_filter_patterns(config.get("URL_DENYLIST", ""))
 
     def url_passes_filters(self, url: str, *, snapshot=None, use_effective_config: bool = True) -> bool:
@@ -678,9 +723,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         return len(urls)
 
     def remaining_url_capacity(self) -> int | None:
-        from archivebox.config.common import get_config
-
-        max_urls = int(get_config(crawl=self).CRAWL_MAX_URLS or 0)
+        max_urls = int(self._config_value(self.get_current_config(refresh=True), "CRAWL_MAX_URLS", 0) or 0)
         if max_urls <= 0:
             return None
         return max(max_urls - self.count_urls_for_limit(), 0)
@@ -690,9 +733,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         return remaining is None or remaining > 0
 
     def remaining_snapshot_capacity(self) -> int | None:
-        from archivebox.config.common import get_config
-
-        max_urls = int(get_config(crawl=self).CRAWL_MAX_URLS or 0)
+        max_urls = int(self._config_value(self.get_current_config(refresh=True), "CRAWL_MAX_URLS", 0) or 0)
         if max_urls <= 0:
             return None
         return max(max_urls - self.snapshot_set.count(), 0)
@@ -922,21 +963,27 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             List of newly created Snapshot objects
         """
         from archivebox.core.models import Snapshot, Tag
-        from archivebox.config.common import get_config
         from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
 
         if self.status == self.StatusChoices.SEALED:
             return []
+        # Internal-input crawls preserve the submitted text verbatim in
+        # Crawl.urls. The root snapshot's parser hooks are the only supported
+        # path for turning that text into child snapshots, otherwise a later
+        # runner pass could reinterpret plain URL-looking lines as direct
+        # depth-0 work and bypass format-specific metadata parsing.
+        if self.has_internal_input_root():
+            return []
 
         created_snapshots = []
         crawl_tag_names = self.current_tag_names()
         tags_by_name: dict[str, Tag] = {}
-        config = get_config(crawl=self)
-        only_new_urls = bool(config.ONLY_NEW)
 
         for line in self.urls.splitlines():
             if not line.strip():
                 continue
+            config = self.get_current_config(refresh=True)
+            only_new_urls = bool(self._config_value(config, "ONLY_NEW", True))
 
             # Parse JSONL or plain URL
             try:
@@ -962,10 +1009,10 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             except ValueError as err:
                 print(f"[yellow][!] Skipping invalid snapshot URL: {url[:120]}... ({err})[/yellow]")
                 continue
-            if Snapshot.is_archivebox_internal_url(url):
+            if Snapshot.is_archivebox_internal_url(url, config=config):
                 print(f"[yellow][!] Skipping internal ArchiveBox snapshot URL: {url}[/yellow]")
                 continue
-            if not self.url_passes_filters(url):
+            if not self.url_passes_filters(url, use_effective_config=False):
                 continue
             if only_new_urls and Snapshot.objects.filter(url=url).exists():
                 continue
@@ -1073,7 +1120,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     ) -> list["Snapshot"]:
         """Create child snapshots from discovered URL records after filtering and deduping once."""
         from archivebox.core.models import Snapshot, SnapshotTag, Tag
-        from archivebox.config.common import get_config
         from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
 
         if self.status == self.StatusChoices.SEALED:
@@ -1083,30 +1129,71 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             return []
 
         crawl_tag_names = self.current_tag_names()
-        config = get_config(crawl=self, snapshot=parent_snapshot)
+        config = self.get_current_config(refresh=True)
+        if parent_snapshot is not None and parent_snapshot.config:
+            config.update(parent_snapshot.config)
         allowlist = self.split_filter_patterns(config.get("URL_ALLOWLIST", ""))
         denylist = self.split_filter_patterns(config.get("URL_DENYLIST", ""))
 
+        def metadata_score(record: Mapping[str, Any]) -> int:
+            # Multiple parsers can discover the same URL from one import root.
+            # Keep the record with the richest user-facing metadata so generic
+            # text/HTML extraction does not erase RSS/Netscape/JSON fields.
+            return sum(bool(record.get(field)) for field in ("title", "bookmarked_at", "timestamp", "tags"))
+
         deduped_records: dict[str, Mapping[str, Any]] = {}
         for record in records:
             url = sanitize_extracted_url(fix_url_from_markdown(str(record.get("url") or "").strip()))
-            if not url or url in deduped_records:
+            if not url:
                 continue
             try:
                 validate_url(url)
             except ValueError as err:
                 print(f"[yellow][!] Skipping invalid discovered snapshot URL: {url[:120]}... ({err})[/yellow]")
                 continue
-            if Snapshot.is_archivebox_internal_url(url):
+            if Snapshot.is_archivebox_internal_url(url, config=config):
                 print(f"[yellow][!] Skipping internal ArchiveBox discovered snapshot URL: {url}[/yellow]")
                 continue
             if self.url_passes_compiled_filters(url, allowlist=allowlist, denylist=denylist):
-                deduped_records[url] = record
+                existing_record = deduped_records.get(url)
+                if existing_record is None or metadata_score(record) > metadata_score(existing_record):
+                    deduped_records[url] = record
 
         if not deduped_records:
             return []
 
-        existing_scope = Snapshot.objects if bool(config.ONLY_NEW) else self.snapshot_set
+        existing_in_crawl = {
+            snapshot.url: snapshot for snapshot in self.snapshot_set.prefetch_related("tags").filter(url__in=deduped_records.keys())
+        }
+        for url, snapshot in existing_in_crawl.items():
+            record = deduped_records[url]
+            update_fields = []
+            title = Snapshot._normalize_title_candidate(str(record.get("title") or "").strip()[:512], snapshot_url=url)
+            if title and (not snapshot.title or len(title) > len(snapshot.title or "")):
+                snapshot.title = title
+                update_fields.append("title")
+            bookmarked_at = None
+            try:
+                bookmarked_at = parse_date(record.get("bookmarked_at") or record.get("timestamp"))
+            except (TypeError, ValueError, OSError):
+                pass
+            if bookmarked_at and snapshot.bookmarked_at != bookmarked_at:
+                snapshot.bookmarked_at = bookmarked_at
+                update_fields.append("bookmarked_at")
+            if update_fields:
+                snapshot.save(update_fields=[*update_fields, "modified_at"])
+            tag_names = {
+                *crawl_tag_names,
+                *self.parse_tag_names(
+                    str(record.get("tags") or ""),
+                    pattern=self._config_value(config, "TAG_SEPARATOR_PATTERN", r"[,]"),
+                ),
+            }
+            if tag_names:
+                tag_ids = [Tag.objects.get_or_create(name=tag_name)[0].pk for tag_name in tag_names]
+                snapshot.tags.add(*tag_ids)
+
+        existing_scope = Snapshot.objects if bool(self._config_value(config, "ONLY_NEW", True)) else self.snapshot_set
         existing_urls = set(existing_scope.filter(url__in=deduped_records.keys()).values_list("url", flat=True))
         urls = [url for url in deduped_records.keys() if url not in existing_urls]
         remaining = self.remaining_snapshot_capacity()
@@ -1116,27 +1203,34 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             return []
 
         now = timezone.now()
-        snapshots = [
-            Snapshot(
-                url=url,
-                timestamp=str((now + timedelta(microseconds=index)).timestamp()),
-                title=Snapshot._normalize_title_candidate(
-                    str(deduped_records[url].get("title") or "").strip()[:512],
-                    snapshot_url=url,
-                )
-                or None,
-                crawl=self,
-                parent_snapshot=parent_snapshot,
-                depth=depth,
-                status=Snapshot.StatusChoices.QUEUED,
-                retry_at=now,
-                bookmarked_at=now,
-                created_at=now,
+        snapshots = []
+        for index, url in enumerate(urls):
+            record = deduped_records[url]
+            bookmarked_at = now
+            try:
+                bookmarked_at = parse_date(record.get("bookmarked_at") or record.get("timestamp")) or now
+            except (TypeError, ValueError, OSError):
+                pass
+            snapshots.append(
+                Snapshot(
+                    url=url,
+                    timestamp=str((now + timedelta(microseconds=index)).timestamp()),
+                    title=Snapshot._normalize_title_candidate(
+                        str(record.get("title") or "").strip()[:512],
+                        snapshot_url=url,
+                    )
+                    or None,
+                    crawl=self,
+                    parent_snapshot=parent_snapshot,
+                    depth=depth,
+                    status=Snapshot.StatusChoices.QUEUED,
+                    retry_at=now,
+                    bookmarked_at=bookmarked_at,
+                    created_at=now,
+                ),
             )
-            for index, url in enumerate(urls)
-        ]
         for snapshot in snapshots:
-            snapshot.set_delete_at_from_config(config.DELETE_AFTER)
+            snapshot.set_delete_at_from_config(self._config_value(config, "DELETE_AFTER", "0"))
 
         created_snapshots = []
         for snapshot in snapshots:
@@ -1155,7 +1249,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         crawl_urls = {url for _raw_line, url in self._iter_url_lines() if url}
         new_url_lines = [snapshot.url for snapshot in created_snapshots if snapshot.url not in crawl_urls]
-        if new_url_lines:
+        # For internal-input crawls, Crawl.urls is the immutable source text.
+        # Child snapshots are the parsed/indexed representation, so appending
+        # discovered URLs here would both duplicate state and destroy the exact
+        # import artifact users submitted through CLI/API/UI.
+        if new_url_lines and not self.has_internal_input_root():
             self.urls = (self.urls.rstrip() + "\n" + "\n".join(new_url_lines)).lstrip("\n")
             self.save(update_fields=["urls", "modified_at"])
 
@@ -1163,7 +1261,10 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         for snapshot in created_snapshots:
             tag_names = {
                 *crawl_tag_names,
-                *self.parse_tag_names(str(deduped_records[snapshot.url].get("tags") or ""), pattern=config.TAG_SEPARATOR_PATTERN),
+                *self.parse_tag_names(
+                    str(deduped_records[snapshot.url].get("tags") or ""),
+                    pattern=self._config_value(config, "TAG_SEPARATOR_PATTERN", r"[,]"),
+                ),
             }
             if tag_names:
                 tag_names_by_url[snapshot.url] = tag_names
@@ -1286,7 +1387,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             The root Snapshot for this crawl, or None for system crawls that don't create snapshots
         """
         import time
-        from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records, is_finite_background_hook
+        from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records
         from archivebox.config.common import get_config
         from archivebox.machine.models import Binary, Machine
 
@@ -1344,19 +1445,15 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 print(f"[yellow]⏱️  Hook {hook.name} took {hook_elapsed:.2f}s[/yellow]")
 
             if process.status == process.StatusChoices.RUNNING:
-                if not is_finite_background_hook(hook.name):
-                    return set()
-                try:
-                    process.wait(timeout=process.timeout)
-                except Exception:
+                if process.poll() is None:
                     return set()
 
             from archivebox.plugins.hooks import extract_records_from_process
 
             records = []
-            # Finite background hooks can exit before their completed Process
-            # metadata is visible. Give successful hooks a brief chance to
-            # flush JSONL stdout into the Process row before downstream hooks.
+            # A hook can exit before its completed Process metadata is visible.
+            # Give successful hooks a brief chance to flush JSONL stdout into
+            # the Process row before downstream hooks.
             for delay in (0.0, 0.05, 0.1, 0.25, 0.5):
                 if delay:
                     time.sleep(delay)

archivebox/machine/admin.py

@@ -434,6 +434,9 @@ class ProcessAdmin(BaseModelAdmin):
         "crawl_link",
         "cmd_display",
         "env_display",
+        "stdout_display",
+        "stderr_display",
+        "archiveresult_output_display",
         "timeout",
         "pid",
         "exit_code",
@@ -475,7 +478,7 @@ class ProcessAdmin(BaseModelAdmin):
         (
             "Output",
             {
-                "fields": ("stdout", "stderr"),
+                "fields": ("stdout_display", "stderr_display", "archiveresult_output_display"),
                 "classes": ("card", "wide", "collapse"),
             },
         ),
@@ -548,6 +551,7 @@ class ProcessAdmin(BaseModelAdmin):
         label="Kill",
         description="Kill this process if it is still running",
         attrs={"class": "deletelink"},
+        methods=("POST",),
     )
     def kill_process(self, request, obj):
         self._terminate_processes(request, [obj])
@@ -739,6 +743,28 @@ class ProcessAdmin(BaseModelAdmin):
             return "-"
         return _render_copy_block(env_text, multiline=True)
 
+    @admin.display(description="Stdout")
+    def stdout_display(self, process):
+        if not process.stdout:
+            return "-"
+        return _render_copy_block(process.stdout, multiline=True)
+
+    @admin.display(description="Stderr")
+    def stderr_display(self, process):
+        if not process.stderr:
+            return "-"
+        return _render_copy_block(process.stderr, multiline=True)
+
+    @admin.display(description="ArchiveResult Output")
+    def archiveresult_output_display(self, process):
+        try:
+            output = process.archiveresult.output_str
+        except Process.archiveresult.RelatedObjectDoesNotExist:
+            return "-"
+        if not output:
+            return "-"
+        return _render_copy_block(output, multiline=True)
+
 
 def register_admin(admin_site):
     admin_site.register(Machine, MachineAdmin)

archivebox/machine/migrations/0020_repair_process_binary_iface_links.py

@@ -0,0 +1,130 @@
+import json
+from pathlib import Path
+
+from django.db import migrations
+from django.utils import timezone
+
+
+def _cmd_array(value):
+    if isinstance(value, list):
+        return value
+    if isinstance(value, str):
+        try:
+            parsed = json.loads(value)
+        except json.JSONDecodeError:
+            return [value] if value else []
+        return parsed if isinstance(parsed, list) else []
+    return []
+
+
+def _ensure_placeholder_iface(NetworkInterface, machine_id, hostname):
+    iface = NetworkInterface.objects.filter(machine_id=machine_id).order_by("-modified_at", "-created_at").first()
+    if iface is not None:
+        return iface
+
+    now = timezone.now()
+    return NetworkInterface.objects.create(
+        machine_id=machine_id,
+        created_at=now,
+        modified_at=now,
+        mac_address="00:00:00:00:00:00",
+        ip_public="0.0.0.0",
+        ip_local="0.0.0.0",
+        dns_server="0.0.0.0",
+        hostname=(hostname or "unknown")[:63],
+        iface="unknown",
+        isp="",
+        city="",
+        region="",
+        country="",
+    )
+
+
+def _get_or_create_binary(Binary, machine_id, reference):
+    reference = str(reference or "").strip()
+    if not reference:
+        return None
+
+    name = Path(reference).name or reference
+    qs = Binary.objects.filter(machine_id=machine_id)
+    binary = qs.filter(abspath=reference).order_by("-modified_at", "-created_at").first()
+    if binary is None:
+        binary = qs.filter(name=name).order_by("-modified_at", "-created_at").first()
+    if binary is not None:
+        return binary
+
+    now = timezone.now()
+    return Binary.objects.create(
+        machine_id=machine_id,
+        created_at=now,
+        modified_at=now,
+        name=name[:63],
+        binproviders="env",
+        overrides={},
+        binprovider="env",
+        abspath=reference[:255],
+        version="",
+        sha256="",
+        status="installed",
+        retry_at=None,
+    )
+
+
+def repair_process_binary_iface_links(apps, schema_editor):
+    Binary = apps.get_model("machine", "Binary")
+    Machine = apps.get_model("machine", "Machine")
+    NetworkInterface = apps.get_model("machine", "NetworkInterface")
+    Process = apps.get_model("machine", "Process")
+
+    machines = {machine.id: machine for machine in Machine.objects.only("id", "hostname").iterator(chunk_size=100)}
+    iface_by_machine = {}
+    binary_by_key = {}
+
+    qs = Process.objects.filter(machine_id__isnull=False).filter(binary_id__isnull=True) | Process.objects.filter(
+        machine_id__isnull=False,
+        iface_id__isnull=True,
+    )
+    for process in qs.distinct().only("id", "machine_id", "binary_id", "iface_id", "cmd").iterator(chunk_size=500):
+        update_fields = []
+        machine = machines.get(process.machine_id)
+
+        if process.iface_id is None:
+            iface = iface_by_machine.get(process.machine_id)
+            if iface is None:
+                iface = _ensure_placeholder_iface(
+                    NetworkInterface,
+                    process.machine_id,
+                    machine.hostname if machine is not None else "",
+                )
+                iface_by_machine[process.machine_id] = iface
+            process.iface_id = iface.id
+            update_fields.append("iface_id")
+
+        if process.binary_id is None:
+            cmd = _cmd_array(process.cmd)
+            reference = str(cmd[0]).strip() if cmd else ""
+            if reference:
+                key = (process.machine_id, reference)
+                binary = binary_by_key.get(key)
+                if binary is None:
+                    binary = _get_or_create_binary(Binary, process.machine_id, reference)
+                    binary_by_key[key] = binary
+                if binary is not None:
+                    process.binary_id = binary.id
+                    update_fields.append("binary_id")
+
+        if update_fields:
+            process.modified_at = timezone.now()
+            process.save(update_fields=[*update_fields, "modified_at"])
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("machine", "0019_single_active_runner_constraint"),
+    ]
+
+    operations = [
+        migrations.RunPython(repair_process_binary_iface_links, migrations.RunPython.noop),
+    ]

archivebox/machine/models.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 __package__ = "archivebox.machine"
 
 import os
+import signal
 import sys
 import uuid
 import socket
@@ -55,6 +56,13 @@ PROCESS_TIMEOUT_GRACE = timedelta(seconds=30)  # Extra margin before force-clean
 START_TIME_TOLERANCE = 5.0  # Seconds tolerance for start time matching
 
 
+def _default_exit_code_for_unowned_process(process_type: str) -> int:
+    # Hooks are externally visible work items. If their owning runner disappeared
+    # before recording the real exit code, retrying is safer than converting an
+    # unknown interrupted extraction into a durable success/no-result row.
+    return 128 + signal.SIGTERM if process_type == Process.TypeChoices.HOOK else 0
+
+
 def _find_existing_binary_for_reference(machine: Machine, reference: str) -> Binary | None:
     reference = str(reference or "").strip()
     if not reference:
@@ -650,22 +658,6 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
         binary_overrides = record.get("overrides", {})
         normalized_overrides = binary_overrides if isinstance(binary_overrides, dict) else {}
 
-        # abx-plugins currently emits a GitHub install URL for readability-extractor,
-        # but the package is published on npm. Prefer the registry package to avoid
-        # long git-based installs in CI while still using canonical install_args.
-        if (
-            name == "readability-extractor"
-            and isinstance(normalized_overrides.get("npm"), dict)
-            and normalized_overrides["npm"].get("install_args") == ["https://github.com/ArchiveBox/readability-extractor"]
-        ):
-            normalized_overrides = {
-                **normalized_overrides,
-                "npm": {
-                    **normalized_overrides["npm"],
-                    "install_args": ["readability-extractor"],
-                },
-            }
-
         # Case 1: Already installed (from on_Crawl hooks) - has abspath AND binproviders
         # This happens when on_Crawl hooks detect already-installed binaries
         abspath = record.get("abspath")
@@ -689,7 +681,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
             )
             from archivebox.config.common import get_config
 
-            binary.symlink_to_lib_bin_after_commit(get_config().LIB_BIN_DIR)
+            binary.symlink_to_lib_bin_after_commit(get_config().LIB_DIR / "bin")
             return binary
 
         # Case 2: From binaries.json - create queued binary (needs installation)
@@ -722,7 +714,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
             )
             from archivebox.config.common import get_config
 
-            binary.symlink_to_lib_bin_after_commit(get_config().LIB_BIN_DIR)
+            binary.symlink_to_lib_bin_after_commit(get_config().LIB_DIR / "bin")
             return binary
 
         return None
@@ -758,7 +750,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
 
     def symlink_to_lib_bin(self, lib_bin_dir: str | Path) -> Path | None:
         """
-        Symlink this binary into LIB_BIN_DIR for human-facing convenience.
+        Symlink this binary into a derived lib/bin directory for human-facing convenience.
 
         After a binary is installed by any binprovider (pip, npm, brew, apt, etc),
         we can optionally expose a flat convenience directory for shell users.
@@ -766,7 +758,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
         paths, not this indirection.
 
         Args:
-            lib_bin_dir: Path to LIB_BIN_DIR (e.g., /data/lib/arm64-darwin/bin)
+            lib_bin_dir: Path to the derived convenience bin dir (e.g., /data/lib/bin)
 
         Returns:
             Path to the created symlink, or None if symlinking failed
@@ -790,14 +782,14 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
         except StopIteration:
             app_index = -1
 
-        # Create LIB_BIN_DIR if it doesn't exist
+        # Create the derived convenience bin dir if it doesn't exist.
         try:
             lib_bin_dir.mkdir(parents=True, exist_ok=True)
         except (OSError, PermissionError) as e:
-            print(f"Failed to create LIB_BIN_DIR {lib_bin_dir}: {e}", file=sys.stderr)
+            print(f"Failed to create lib/bin convenience dir {lib_bin_dir}: {e}", file=sys.stderr)
             return None
 
-        # Expose the canonical Binary.name in LIB_BIN_DIR. Some providers point
+        # Expose the canonical Binary.name in the convenience bin dir. Some providers point
         # abspath at implementation files like cli.js or manifest.json; those
         # are valid targets, but they are not user-facing binary names.
         binary_name = _canonical_binary_name(self.name) or binary_abspath.name
@@ -1552,9 +1544,11 @@ class Process(ModelWithDeleteAfter, models.Model):
         avoids any race window where the row briefly exists as ADD before a
         higher-level demotion.
         """
-        argv_str = " ".join(sys.argv).lower()
+        argv = [str(arg) for arg in sys.argv]
+        argv_str = " ".join(argv).lower()
+        executable = Path(argv[0]).name.lower() if argv else ""
 
-        if "supervisord" in argv_str:
+        if executable == "supervisord" or any(arg.startswith("--configuration=") for arg in argv[1:]):
             return cls.TypeChoices.SUPERVISORD
         elif "runner_watch" in argv_str:
             return cls.TypeChoices.WORKER
@@ -1624,7 +1618,9 @@ class Process(ModelWithDeleteAfter, models.Model):
                     is_stale = True  # Process no longer exists
 
             if is_stale:
-                proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+                proc.mark_exited(
+                    exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),
+                )
                 cleaned += 1
 
         return cleaned
@@ -2154,8 +2150,7 @@ class Process(ModelWithDeleteAfter, models.Model):
                 # TODO: Uncomment to cleanup (keeping for debugging for now)
                 # self.stderr_file.unlink(missing_ok=True)
 
-            # Try to get exit code from proc or default to unknown
-            self.exit_code = self.exit_code if self.exit_code is not None else 0
+            self.exit_code = self.exit_code if self.exit_code is not None else _default_exit_code_for_unowned_process(self.process_type)
             if self.exit_code == -1:
                 self.exit_code = 137
             self.ended_at = timezone.now()
@@ -2519,7 +2514,9 @@ class Process(ModelWithDeleteAfter, models.Model):
         # orphaned Process backlog cannot be materialized in memory at once.
         for proc in running_children.iterator(chunk_size=100):
             if not proc.is_running:
-                proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+                proc.mark_exited(
+                    exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),
+                )
                 cleaned += 1
                 continue
 
@@ -2543,7 +2540,9 @@ class Process(ModelWithDeleteAfter, models.Model):
             ):
                 continue
 
-            proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+            proc.mark_exited(
+                exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),
+            )
             cleaned += 1
 
         if cleaned:

archivebox/misc/checks.py

@@ -220,14 +220,14 @@ def check_not_inside_source_dir():
 def check_data_dir_permissions(config=None, **config_kwargs):
     from archivebox import DATA_DIR
     from archivebox.misc.logging import STDERR
-    from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, DEFAULT_PUID, DEFAULT_PGID, IS_ROOT, USER
+    from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, DEFAULT_UID, DEFAULT_GID, IS_ROOT, USER
     from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir
 
     data_dir_stat = Path(DATA_DIR).stat()
     data_dir_uid, data_dir_gid = data_dir_stat.st_uid, data_dir_stat.st_gid
     data_owned_by_root = data_dir_uid == 0
 
-    # data_owned_by_default_user = data_dir_uid == DEFAULT_PUID or data_dir_gid == DEFAULT_PGID
+    # data_owned_by_default_user = data_dir_uid == DEFAULT_UID or data_dir_gid == DEFAULT_GID
     data_owner_doesnt_match = (data_dir_uid != ARCHIVEBOX_USER and data_dir_gid != ARCHIVEBOX_GROUP) if not IS_ROOT else False
     data_not_writable = not (os.path.isdir(DATA_DIR) and os.access(DATA_DIR, os.W_OK))
     if data_not_writable:
@@ -245,9 +245,9 @@ def check_data_dir_permissions(config=None, **config_kwargs):
 
     if data_not_writable:
         STDERR.print(
-            f"[violet]Hint:[/violet] Change the current ownership [red]{data_dir_uid}[/red]:{data_dir_gid} (PUID:PGID) to the user & group that will run ArchiveBox, e.g.:",
+            f"[violet]Hint:[/violet] Change the current ownership [red]{data_dir_uid}[/red]:{data_dir_gid} to the user & group that will run ArchiveBox, e.g.:",
         )
-        STDERR.print(f"    [grey53]sudo[/grey53] chown -R [blue]{DEFAULT_PUID}:{DEFAULT_PGID}[/blue] {DATA_DIR.resolve()}")
+        STDERR.print(f"    [grey53]sudo[/grey53] chown -R [blue]{DEFAULT_UID}:{DEFAULT_GID}[/blue] {DATA_DIR.resolve()}")
         STDERR.print("    Avoid recursive chown on very large archives unless you know the full tree needs repair.")
         STDERR.print()
         STDERR.print("[blue]More info:[/blue]")
@@ -289,7 +289,14 @@ def check_data_dir_permissions(config=None, **config_kwargs):
 
 
 def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, config=None, **config_kwargs):
-    from archivebox.config.paths import assert_dir_can_contain_unix_sockets, dir_is_writable, get_or_create_working_tmp_dir
+    from archivebox.config.paths import (
+        MAX_TMP_SOCKET_URL_LENGTH,
+        SUPERVISORD_SOCKET_FILENAME,
+        assert_dir_can_contain_unix_sockets,
+        dir_is_writable,
+        get_or_create_working_tmp_dir,
+        tmp_dir_socket_path_is_short_enough,
+    )
     from archivebox.misc.logging import STDERR
     from archivebox.misc.logging_util import pretty_path
     from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
@@ -297,20 +304,22 @@ def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, confi
 
     config = config or get_config(**config_kwargs)
     tmp_dir = tmp_dir or config.TMP_DIR
-    socket_file = tmp_dir.absolute().resolve() / "supervisord.sock"
+    socket_file = tmp_dir.absolute().resolve() / SUPERVISORD_SOCKET_FILENAME
 
     if not must_exist and not os.path.isdir(tmp_dir):
         # just check that its viable based on its length (because dir may not exist yet, we cant check if its writable)
-        return len(f"file://{socket_file}") <= 96
+        return tmp_dir_socket_path_is_short_enough(tmp_dir)
 
     tmp_is_valid = False
     try:
         tmp_is_valid = dir_is_writable(tmp_dir)
         if not config.ALLOW_NO_UNIX_SOCKETS:
             tmp_is_valid = tmp_is_valid and assert_dir_can_contain_unix_sockets(tmp_dir)
-        assert tmp_is_valid, f"ArchiveBox user PUID={ARCHIVEBOX_USER} PGID={ARCHIVEBOX_GROUP} is unable to write to TMP_DIR={tmp_dir}"
-        assert len(f"file://{socket_file}") <= 96, (
-            f"ArchiveBox TMP_DIR={tmp_dir} is too long, dir containing unix socket files must be <90 chars."
+        assert tmp_is_valid, f"ArchiveBox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} is unable to write to TMP_DIR={tmp_dir}"
+        socket_url_len = len(f"file://{socket_file}")
+        assert tmp_dir_socket_path_is_short_enough(tmp_dir), (
+            f"ArchiveBox TMP_DIR={tmp_dir} is too long, file://{socket_file} is {socket_url_len} chars "
+            f"and must be <{MAX_TMP_SOCKET_URL_LENGTH} chars."
         )
         return True
     except Exception as e:
@@ -324,7 +333,7 @@ def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, confi
                     "",
                     "[blue]Info:[/blue] [grey53]The TMP_DIR is used for the supervisord unix socket file and other temporary files.",
                     "  - It [red]must[/red] be on a local drive (not inside a docker volume, remote network drive, or FUSE mount).",
-                    f"  - It [red]must[/red] be readable and writable by the ArchiveBox user (PUID={ARCHIVEBOX_USER}, PGID={ARCHIVEBOX_GROUP}).",
+                    f"  - It [red]must[/red] be readable and writable by the ArchiveBox user ({ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}).",
                     "  - It [red]must[/red] be a *short* path (less than 90 characters) due to UNIX path length restrictions for sockets.",
                     "  - It [yellow]should[/yellow] be able to hold at least 200MB of data (in-progress downloads can be large).[/grey53]",
                     "",
@@ -364,7 +373,7 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
     lib_is_valid = False
     try:
         lib_is_valid = dir_is_writable(lib_dir)
-        assert lib_is_valid, f"ArchiveBox user PUID={ARCHIVEBOX_USER} PGID={ARCHIVEBOX_GROUP} is unable to write to LIB_DIR={lib_dir}"
+        assert lib_is_valid, f"ArchiveBox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} is unable to write to LIB_DIR={lib_dir}"
         return True
     except Exception as e:
         if not quiet:
@@ -376,7 +385,7 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
                     f"   [yellow]{e}[/yellow]",
                     "",
                     "[blue]Info:[/blue] [grey53]The LIB_DIR is used to store ArchiveBox auto-installed plugin library and binary dependencies.",
-                    f"  - It [red]must[/red] be readable and writable by the ArchiveBox user (PUID={ARCHIVEBOX_USER}, PGID={ARCHIVEBOX_GROUP}).",
+                    f"  - It [red]must[/red] be readable and writable by the ArchiveBox user ({ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}).",
                     "  - It [yellow]should[/yellow] be on a local (ideally fast) drive like an SSD or HDD (not on a network drive or external HDD).",
                     "  - It [yellow]should[/yellow] be able to hold at least 1GB of data (some dependencies like Chrome can be large).[/grey53]",
                     "",

archivebox/misc/jsonl.py

@@ -18,6 +18,9 @@ Typed JSONL Format:
 Plain URLs (also supported):
     https://example.com
     https://foo.com
+
+Local filesystem paths and file:// URLs are intentionally not accepted here.
+Pipe file contents through stdin instead of passing a path/URI as a crawl URL.
 """
 
 __package__ = "archivebox.misc"
@@ -27,7 +30,6 @@ __package__ = "archivebox.misc"
 
 import sys
 import json
-import select
 from typing import Any, TextIO
 from collections.abc import Iterable, Iterator
 from pathlib import Path
@@ -81,7 +83,7 @@ def parse_line(line: str) -> dict[str, Any] | None:
             pass
 
     # Treat as plain URL if it looks like one
-    if line.startswith("http://") or line.startswith("https://") or line.startswith("file://"):
+    if line.startswith("http://") or line.startswith("https://"):
         return {"type": TYPE_SNAPSHOT, "url": line}
 
     # Could be a snapshot ID (UUID with dashes or compact 32-char hex)
@@ -112,14 +114,6 @@ def read_stdin(stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
     if active_stream.isatty():
         return
 
-    try:
-        ready, _, _ = select.select([active_stream], [], [], 0)
-    except (OSError, ValueError):
-        ready = [active_stream]
-
-    if not ready:
-        return
-
     for line in active_stream:
         record = parse_line(line)
         if record:
@@ -144,17 +138,13 @@ def read_args_or_stdin(args: Iterable[str], stream: TextIO | None = None) -> Ite
     Read from CLI arguments if provided, otherwise from stdin.
 
     Handles both URLs and JSONL from either source.
+    Does not expand local file path arguments; pipe file contents via stdin.
     """
     if args:
         for arg in args:
-            # Check if it's a file path
-            path = Path(arg)
-            if path.exists() and path.is_file():
-                yield from read_file(path)
-            else:
-                record = parse_line(arg)
-                if record:
-                    yield record
+            record = parse_line(arg)
+            if record:
+                yield record
     else:
         yield from read_stdin(stream)
 

archivebox/misc/paginators.py

@@ -1,63 +1,12 @@
 __package__ = "archivebox.misc"
 
 from django.core.paginator import Paginator
-from django.core.paginator import Page
-from django.core.paginator import EmptyPage
-from django.db import connection
 from django.db.models import QuerySet
 from django.utils.functional import cached_property
 
 
-class CountlessPage(Page):
-    def __init__(self, object_list, number, paginator, has_next_page=False):
-        self.has_next_page = has_next_page
-        super().__init__(object_list, number, paginator)
-
-    def has_next(self):
-        return self.has_next_page
-
-
-class CountlessPaginator(Paginator):
-    has_exact_count = False
-
-    def __init__(self, *args, **kwargs):
-        self._count_hint = 0
-        super().__init__(*args, **kwargs)
-
-    @cached_property
-    def count(self):
-        return self._count_hint
-
-    @cached_property
-    def num_pages(self):
-        return self.count
-
-    def validate_number(self, number):
-        try:
-            number = int(number)
-        except (TypeError, ValueError):
-            raise EmptyPage("That page number is not an integer")
-        if number < 1:
-            raise EmptyPage("That page number is less than 1")
-        return number
-
-    def page(self, number):
-        number = self.validate_number(number)
-        bottom = (number - 1) * self.per_page
-        page_objects = list(self.object_list[bottom : bottom + self.per_page + 1])
-        has_next_page = len(page_objects) > self.per_page
-        if has_next_page:
-            page_objects = page_objects[: self.per_page]
-        self._count_hint = bottom + len(page_objects) + (1 if has_next_page else 0)
-        return CountlessPage(page_objects, number, self, has_next_page=has_next_page)
-
-
 class AcceleratedPaginator(Paginator):
-    """
-    Accelerated paginator ignores DISTINCT when counting total number of rows.
-    Speeds up SELECT Count(*) on Admin views by >20x.
-    https://hakibenita.com/optimizing-the-django-admin-paginator
-    """
+    """Paginator that accepts exact count hints for already-optimized query paths."""
 
     @cached_property
     def count(self):
@@ -69,39 +18,8 @@ class AcceleratedPaginator(Paginator):
         if count_hint is None:
             count_hint = query.__dict__.get("_archivebox_count_hint")
         if count_hint is not None:
-            model = self.object_list.model
-            if count_hint == "model_estimate":
-                return self._model_count_estimate(model)
             if callable(count_hint):
                 return count_hint()
             return count_hint
 
-        if query.distinct or query.where.children:
-            # fallback to normal count method on filtered queryset
-            return super().count
-
-        # otherwise count total rows in a separate fast query
-        return self._model_count_estimate(self.object_list.model)
-
-    def _model_count_estimate(self, model):
-        if connection.vendor == "sqlite":
-            table_name = model._meta.db_table
-            with connection.cursor() as cursor:
-                try:
-                    cursor.execute("SELECT stat FROM sqlite_stat1 WHERE tbl = %s", [table_name])
-                    stats = [int(str(row[0]).split()[0]) for row in cursor.fetchall() if row and row[0]]
-                except Exception:
-                    stats = []
-            if stats:
-                return max(stats)
-
-        return model.objects.count()
-
-        # Alternative approach for PostgreSQL: fallback count takes > 200ms
-        # from django.db import connection, transaction, OperationalError
-        # with transaction.atomic(), connection.cursor() as cursor:
-        #     cursor.execute('SET LOCAL statement_timeout TO 200;')
-        #     try:
-        #         return super().count
-        #     except OperationalError:
-        #         return 9999999999999
+        return super().count

archivebox/misc/serve_static.py

@@ -711,6 +711,10 @@ def _apply_archive_replay_headers(
     return response
 
 
+def _is_asgi_request(request) -> bool:
+    return isinstance(request, ASGIRequest) or "scope" in request.__dict__
+
+
 def serve_static_with_byterange_support(request, path, document_root=None, show_indexes=False, is_archive_replay: bool = False):
     """
     Overrides Django's built-in django.views.static.serve function to support byte range requests.
@@ -718,7 +722,9 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
     https://github.com/satchamo/django/commit/2ce75c5c4bee2a858c0214d136bfcd351fcde11d
     """
     assert document_root
-    config = request.archivebox_config
+    config = request.__dict__.get("archivebox_config")
+    if config is None:
+        config = get_config(resolve_plugins=False)
     fullpath, path = _resolve_archive_path(document_root, path)
     if os.access(fullpath, os.R_OK) and fullpath.is_dir():
         if request.GET.get("download") == "zip" and show_indexes:
@@ -726,7 +732,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 fullpath,
                 path,
                 is_archive_replay=is_archive_replay,
-                use_async_stream=isinstance(request, ASGIRequest),
+                use_async_stream=_is_asgi_request(request),
                 config=config,
             )
         if show_indexes:
@@ -908,7 +914,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             response.headers["Content-Security-Policy"] = (
                 "default-src 'self' data: blob:; "
                 "script-src 'unsafe-inline'; "
-                "style-src 'unsafe-inline'; "
+                "style-src 'unsafe-inline' data: blob:; "
                 "connect-src 'self'; "
                 "frame-src 'self' data: blob:; "
                 "object-src 'none'; "
@@ -977,7 +983,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
     # setup response object
     ranged_file = RangedFileReader(open(fullpath, "rb"))
     response = StreamingHttpResponse(
-        _stream_ranged_file_async(ranged_file) if isinstance(request, ASGIRequest) else ranged_file,
+        _stream_ranged_file_async(ranged_file) if _is_asgi_request(request) else ranged_file,
         content_type=content_type,
     )
     response.headers["Last-Modified"] = http_date(statobj.st_mtime)

archivebox/misc/util.py

@@ -86,6 +86,11 @@ URL_REGEX = re.compile(
     re.IGNORECASE | re.UNICODE,
 )
 
+# Maximum supported URL length. Very long URLs are rare but must be supported correctly
+# (e.g. data: URLs, deeply nested query strings). The Snapshot.url column is stored as a
+# variable-length TextField (so short URLs don't reserve space and very long URLs still
+# fit) while keeping a normal index on the field so exact, prefix, and substring lookups
+# all keep working.
 MAX_URL_LENGTH = 65535
 
 QUOTE_DELIMITERS = (

archivebox/personas/forms.py

@@ -62,7 +62,7 @@ class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm):
         label="Absolute path or CDP URL",
         widget=forms.TextInput(
             attrs={
-                "placeholder": "/Users/alice/Library/Application Support/Google/Chrome  or  ws://127.0.0.1:9222/devtools/browser/...",
+                "placeholder": "/Users/alice/Library/Application Support/Google/Chrome  or  http://127.0.0.1:9222  or  ws://127.0.0.1:9222/devtools/browser/...",
                 "style": "width: 100%; font-family: monospace;",
             },
         ),

archivebox/personas/importers.py

@@ -378,7 +378,6 @@ def discover_persona_template_profiles(personas_dir: Path | None = None) -> list
         candidate_roots.extend(
             [
                 CONSTANTS.PERSONAS_DIR.expanduser(),
-                Path.home() / ".config" / "abx" / "personas",
             ],
         )
 
@@ -612,7 +611,7 @@ def export_browser_state(
     if not state_script.exists():
         return False, None, f"Browser state export script not found at {state_script}"
 
-    node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules"
+    node_modules_dir = get_config().LIB_DIR / "pnpm" / "packages" / "chrome" / "node_modules"
     chrome_plugin_dir = Path(get_plugins_dir()).resolve()
 
     env = os.environ.copy()

archivebox/plugins/forms.py

@@ -9,6 +9,7 @@ from typing import Any
 from django import forms
 from django.utils.html import format_html
 
+from archivebox.config import CONSTANTS_CONFIG
 from archivebox.config.common import ArchiveBoxConfig, get_config
 from archivebox.plugins.discovery import discover_plugin_configs, get_plugin_icon, get_plugins
 
@@ -129,6 +130,7 @@ HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
     "env",
     "media",
     "npm",
+    "opencode",
     "pip",
     "puppeteer",
     "search_backend_ripgrep",
@@ -317,6 +319,7 @@ class PluginConfigFormMixin:
                 for config_key, prop_schema in properties.items()
                 if (
                     isinstance(prop_schema, dict)
+                    and str(config_key) not in CONSTANTS_CONFIG
                     and (self.allow_crawl_execution_config_fields or ArchiveBoxConfig.scope_for_key(str(config_key)) == "crawl_frozen")
                 )
             ]
@@ -431,6 +434,8 @@ class PluginConfigFormMixin:
                 input_name = _plugin_config_input_name(plugin_name, config_key)
                 if input_name not in self.data:
                     continue
+                if str(config_key) in CONSTANTS_CONFIG:
+                    continue
                 if not self.allow_crawl_execution_config_fields and ArchiveBoxConfig.scope_for_key(str(config_key)) != "crawl_frozen":
                     continue
 

archivebox/plugins/hooks.py

@@ -32,7 +32,7 @@ Execution order:
     - After all foreground hooks complete, background hooks receive SIGTERM and must finalize
 
 Hook naming convention:
-    on_{EventFamily}__{run_order}_{description}[.finite.bg|.daemon.bg].{ext}
+    on_{EventFamily}__{run_order}_{description}[.bg].{ext}
 
 API:
     discover_hooks(event)     -> List[Path]     Find hook scripts for a hook-backed event family
@@ -104,11 +104,6 @@ def is_background_hook(hook_name: str) -> bool:
     return ".bg." in hook_name or "__background" in hook_name
 
 
-def is_finite_background_hook(hook_name: str) -> bool:
-    """Check if a background hook is finite-lived and should be awaited."""
-    return ".finite.bg." in hook_name
-
-
 def normalize_hook_event_name(event_name: str) -> str | None:
     """
     Normalize a hook event family or event class name to its on_* prefix.
@@ -285,7 +280,6 @@ def run_hook(
     """
     from archivebox.machine.models import Process, Machine, NetworkInterface
     from archivebox.config.common import get_config, normalize_runtime_config
-    import sys
 
     config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
     config_overrides = _config_to_overrides(config)
@@ -333,12 +327,15 @@ def run_hook(
         )
         return process
 
-    # Determine the interpreter based on file extension
+    # Python hooks carry their runtime contract in the shebang
+    # (usually `abxpkg run --script python3`), so execute them directly.
+    # For shell/JS hooks we still dispatch through the conventional
+    # interpreter because those hooks do not need per-script Python env setup.
     ext = script.suffix.lower()
     if ext == ".sh":
         cmd = ["bash", str(script)]
     elif ext == ".py":
-        cmd = [sys.executable, str(script)]
+        cmd = [str(script)]
     elif ext == ".js":
         cmd = ["node", str(script)]
     else:
@@ -379,18 +376,18 @@ def run_hook(
         env["CRAWL_DIR"] = str(crawl_dir)
 
     # Export runtime library roots; abx-dl/abxpkg own executable lookup env.
-    lib_dir = resolved_config.LIB_DIR
+    lib_dir = hook_config.get("LIB_DIR")
     if lib_dir:
         env["LIB_DIR"] = str(lib_dir)
         env["ABXPKG_LIB_DIR"] = str(lib_dir)
 
     # Set Node.js module resolution paths.
     # NODE_PATH may be a path list, but NODE_MODULES_DIR is a single canonical directory.
-    node_modules_dir = resolved_config.get("NODE_MODULES_DIR")
+    node_modules_dir = hook_config.get("NODE_MODULES_DIR")
     if not node_modules_dir and lib_dir:
-        node_modules_dir = Path(lib_dir) / "npm" / "node_modules"
+        node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"
 
-    node_path_parts = [part for part in str(resolved_config.get("NODE_PATH") or "").split(os.pathsep) if part]
+    node_path_parts = [part for part in str(hook_config.get("NODE_PATH") or "").split(os.pathsep) if part]
     if node_modules_dir:
         node_modules_dir = Path(node_modules_dir)
         node_modules_dir.mkdir(parents=True, exist_ok=True)
@@ -408,7 +405,6 @@ def run_hook(
         "PATH",
         "LIB_DIR",
         "ABXPKG_LIB_DIR",
-        "LIB_BIN_DIR",
         "NODE_PATH",
         "NODE_MODULES_DIR",
         "NODE_MODULE_DIR",
@@ -432,8 +428,8 @@ def run_hook(
     # Create output directory if needed
     output_dir.mkdir(parents=True, exist_ok=True)
 
-    # Detect if this is a background hook (long-running daemon)
-    # Background hooks use the .daemon.bg. or .finite.bg. filename convention.
+    # Detect if this is a background hook.
+    # Background hooks use the .bg. filename marker.
     # Old convention: __background in stem (for backwards compatibility)
     is_background = ".bg." in script.name or "__background" in script.stem
 

archivebox/plugins/views.py

@@ -14,13 +14,13 @@ from django.utils.safestring import mark_safe
 from admin_data_views.typing import ItemContext, SectionData, TableContext
 from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view
 
+from archivebox.config.common import get_live_config_url
 from archivebox.config.views import get_environment_binary_url, is_superuser
 from archivebox.plugins.discovery import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, discover_plugin_configs, iter_plugin_dirs
 
 
 ABX_PLUGINS_DOCS_BASE_URL = "https://archivebox.github.io/abx-plugins/"
 ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/"
-LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 LIVE_PLUGIN_BASE_URL = "/admin/environment/plugins/"
 
 
@@ -73,18 +73,10 @@ def get_plugin_hook_source_url(plugin_name: str, hook_name: str) -> str:
     return f"{ABX_PLUGINS_GITHUB_BASE_URL}{quote(plugin_name)}/{quote(hook_name)}"
 
 
-def get_live_config_url(key: str) -> str:
-    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
-
-
 def get_machine_admin_url() -> str | None:
-    try:
-        from archivebox.machine.models import Machine
+    from archivebox.machine.models import Machine
 
-        machine = Machine.current()
-        return machine.admin_change_url or f"/admin/machine/machine/{machine.id}/change/"
-    except Exception:
-        return None
+    return Machine.current().admin_change_url
 
 
 def render_code_tag_list(values: list[str]) -> str:

archivebox/progressmonitor/templates/progressmonitor/progress_monitor.html

@@ -35,6 +35,9 @@
         top: 0;
         z-index: 10;
     }
+    #progress-monitor.collapsed .header-bar {
+        cursor: pointer;
+    }
     #progress-monitor .header-left {
         display: flex;
         align-items: center;
@@ -71,7 +74,12 @@
         animation: idle-pulse 5s infinite;
     }
     #progress-monitor .status-dot.stopped {
-        background: #6e7681;
+        background: #d29922;
+        box-shadow: 0 0 4px #d29922;
+    }
+    #progress-monitor .status-dot.error {
+        background: #f85149;
+        box-shadow: 0 0 8px #f85149;
     }
     #progress-monitor .status-dot.flash {
         animation: flash 0.3s ease-out;
@@ -967,19 +975,19 @@
             <div class="stats">
                 <div class="stat">
                     <span class="stat-label">Crawls</span>
-                    <span class="stat-value compact info"><span id="crawls-active">0</span> active · <span id="crawls-queued">0</span> queued</span>
+                    <span class="stat-value compact info"><span id="crawls-active-segment"><span id="crawls-active">0</span> active · </span><span id="crawls-queued">0</span> queued</span>
                 </div>
                 <div class="stat">
                     <span class="stat-label">Snapshots</span>
-                    <span class="stat-value compact info"><span id="snapshots-active">0</span> active · <span id="snapshots-queued">0</span> queued</span>
+                    <span class="stat-value compact info"><span id="snapshots-active-segment"><span id="snapshots-active">0</span> active · </span><span id="snapshots-queued">0</span> queued</span>
                 </div>
                 <div class="stat">
                     <span class="stat-label">Downloads</span>
-                    <span class="stat-value compact warning"><span id="downloads-active">0</span> active · <span id="downloads-queued">0</span> queued</span>
+                    <span class="stat-value compact warning"><span id="downloads-active-segment"><span id="downloads-active">0</span> active · </span><span id="downloads-queued">0</span> queued</span>
                 </div>
                 <div class="stat">
                     <span class="stat-label">Indexing</span>
-                    <span class="stat-value compact success"><span id="indexing-active">0</span> active · <span id="indexing-queued">0</span> queued</span>
+                    <span class="stat-value compact success"><span id="indexing-active-segment"><span id="indexing-active">0</span> active · </span><span id="indexing-queued">0</span> queued</span>
                 </div>
             </div>
         </div>
@@ -1682,17 +1690,24 @@
         `;
     }
 
+    function setOrchestratorState(state, label) {
+        const dot = document.getElementById('orchestrator-dot');
+        dot.classList.remove('stopped', 'idle', 'running', 'error');
+        dot.classList.add(state);
+        document.getElementById('orchestrator-text').textContent = label;
+        return dot;
+    }
+
     function updateProgress(data) {
+        idleMessage.style.color = '';
         function setCount(id, value) {
             const el = document.getElementById(id);
             if (el) el.textContent = Number(value || 0).toLocaleString();
         }
-        function setOrchestratorState(state, label) {
-            const dot = document.getElementById('orchestrator-dot');
-            dot.classList.remove('stopped', 'idle', 'running');
-            dot.classList.add(state);
-            document.getElementById('orchestrator-text').textContent = label;
-            return dot;
+        function setActiveCount(id, value) {
+            setCount(id, value);
+            const segment = document.getElementById(`${id}-segment`);
+            if (segment) segment.style.display = Number(value || 0) > 0 ? '' : 'none';
         }
 
         // Calculate if there's activity
@@ -1715,17 +1730,15 @@
         // Update orchestrator status - show "Running" only when there are active workers.
         const pidEl = document.getElementById('orchestrator-pid');
         const hasWorkers = data.total_workers > 0;
-        const hasBlockedCrawl = (data.active_crawls || []).some(c => c.worker_state === 'crashed');
         let dot = null;
 
         if (hasWorkers) {
             dot = setOrchestratorState('running', 'Running');
-        } else if (hasActivity && hasBlockedCrawl && !data.orchestrator_running) {
+        } else if (!data.orchestrator_running) {
             dot = setOrchestratorState('stopped', 'Runner stopped');
         } else if (hasActivity) {
-            dot = setOrchestratorState('idle', data.orchestrator_running ? 'Idle' : 'Waiting');
+            dot = setOrchestratorState('idle', 'Idle');
         } else {
-            // No activity - show as idle (whether orchestrator process exists or not)
             dot = setOrchestratorState('idle', 'Idle');
         }
 
@@ -1742,15 +1755,17 @@
         setTimeout(() => dot.classList.remove('flash'), 300);
 
         [
-            ['crawls-active', data.crawls_active],
             ['crawls-queued', data.crawls_queued],
-            ['snapshots-active', data.snapshots_active],
             ['snapshots-queued', data.snapshots_queued],
-            ['downloads-active', data.downloads_active],
             ['downloads-queued', data.downloads_queued],
-            ['indexing-active', data.indexing_active],
             ['indexing-queued', data.indexing_queued],
         ].forEach(([id, value]) => setCount(id, value));
+        [
+            ['crawls-active', data.crawls_active],
+            ['snapshots-active', data.snapshots_active],
+            ['downloads-active', data.downloads_active],
+            ['indexing-active', data.indexing_active],
+        ].forEach(([id, value]) => setActiveCount(id, value));
         updateScreencastPanel(data);
 
         // Render crawl tree
@@ -1784,18 +1799,24 @@
 
     function fetchProgress() {
         fetch(progressEndpoint, { credentials: 'same-origin' })
-            .then(response => response.json())
+            .then(response => {
+                if (!response.ok) throw new Error(`HTTP ${response.status}`);
+                return response.json();
+            })
             .then(data => {
                 if (data.error) {
                     console.error('Progress API error:', data.error, data.traceback);
+                    setOrchestratorState('error', 'Backend not responding');
                     idleMessage.textContent = 'API Error: ' + data.error;
                     idleMessage.style.color = '#f85149';
+                    return;
                 }
                 monitor.classList.toggle('is-guest', data.is_admin === false);
                 updateProgress(data);
             })
             .catch(error => {
                 console.error('Progress fetch error:', error);
+                setOrchestratorState('error', 'Backend not responding');
                 idleMessage.textContent = 'Fetch Error: ' + error.message;
                 idleMessage.style.color = '#f85149';
             });
@@ -1866,9 +1887,14 @@
     }
 
     // Collapse toggle
-    collapseBtn.addEventListener('click', function() {
+    collapseBtn.addEventListener('click', function(event) {
+        event.stopPropagation();
         setCollapsedState(!isCollapsed);
     });
+    monitor.querySelector('.header-bar').addEventListener('click', function(event) {
+        if (event.target.closest('button, a, input, select, textarea, [role="button"]')) return;
+        if (isCollapsed) setCollapsedState(false);
+    });
 
     crawlTree.addEventListener('click', function(event) {
         const actionBtn = event.target.closest('.crawl-action-btn');

archivebox/progressmonitor/views.py

@@ -327,8 +327,20 @@ def live_progress_view(request):
             .order_by("-modified_at")[:max_active_crawls],
         )
         paused_crawls = list(
-            crawl_scope.filter(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
+            crawl_scope.filter(
+                Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
+                | Q(
+                    status=Crawl.StatusChoices.PAUSED,
+                    snapshot_set__status__in=Snapshot.RUNNABLE_STATES,
+                    snapshot_set__retry_at__lte=now,
+                )
+                | Q(
+                    status=Crawl.StatusChoices.PAUSED,
+                    snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
+                ),
+            )
             .values(*active_crawl_fields)
+            .distinct()
             .order_by("-modified_at")[:max_active_crawls],
         )
         queued_crawls = list(

archivebox/search/admin.py

@@ -2,6 +2,7 @@ __package__ = "archivebox.search"
 
 from django.contrib import admin
 from django.contrib.admin.views.main import ChangeList
+from django.db.models import Case, IntegerField, Value, When
 
 from archivebox.search.config import (
     get_default_search_mode,
@@ -71,7 +72,13 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
         if queryset.model._meta.label_lower == "core.snapshot" and request.GET.get("_embedded") != "crawl":
             cached_ids = get_cached_admin_search_ids(request)
             if cached_ids is not None:
-                return queryset.filter(pk__in=cached_ids) if cached_ids else queryset.none(), False
+                if not cached_ids:
+                    return queryset.none(), False
+                search_rank = Case(
+                    *(When(pk=snapshot_id, then=Value(index)) for index, snapshot_id in enumerate(cached_ids)),
+                    output_field=IntegerField(),
+                )
+                return queryset.filter(pk__in=cached_ids).annotate(search_rank=search_rank).order_by("search_rank"), False
             return queryset.none(), False
 
         if get_search_mode_base(search_mode, config=request.archivebox_config) == "meta":

archivebox/search/config.py

@@ -62,7 +62,7 @@ def get_search_mode_options(config: dict[str, Any] | None = None, **config_kwarg
     ]
     options = [
         {"value": "meta", "label": "meta"},
-        {"value": "contents", "label": "contents"},
+        {"value": "contents", "label": "deep"},
     ]
     if backend_names:
         options.extend(

archivebox/search/views.py

@@ -4,20 +4,23 @@ import asyncio
 import hashlib
 import json
 import threading
+import time
 from copy import copy
 from queue import Full, Queue
 from urllib.parse import urlsplit
 from uuid import UUID
 
 from django.core.cache import cache
+from django.db import connections
 from django.db.models import Q
-from django.http import QueryDict, StreamingHttpResponse
+from django.http import HttpResponseForbidden, QueryDict, StreamingHttpResponse
 
 from archivebox.search.config import get_search_mode, get_search_mode_base
 from archivebox.search.query import crawl_config_values_search_wave, iter_query_search_ids
 
 
 SEARCH_RESULT_CACHE_TTL = 60
+URL_PREFIX_SEARCH_LIMIT = 500
 
 
 def get_admin_search_cache_key(request, url: str | None = None) -> str:
@@ -34,6 +37,17 @@ def get_admin_search_cache_key(request, url: str | None = None) -> str:
     return f"abx:admin-search:{hashlib.sha256(payload.encode()).hexdigest()}"
 
 
+def get_public_search_cache_key(request, url: str | None = None) -> str:
+    """Build the cache key for one public search URL."""
+    payload = json.dumps(
+        {
+            "url": url or request.get_full_path(),
+        },
+        sort_keys=True,
+    )
+    return f"abx:public-search:{hashlib.sha256(payload.encode()).hexdigest()}"
+
+
 def get_cached_admin_search_ids(request) -> list[str] | None:
     """Return streamed admin search IDs from Django cache."""
     cached = cache.get(get_admin_search_cache_key(request))
@@ -42,7 +56,100 @@ def get_cached_admin_search_ids(request) -> list[str] | None:
     return None
 
 
-def iter_admin_meta_search_ids(query, queryset):
+def get_cached_public_search_ids(request) -> list[str] | None:
+    """Return streamed public search IDs from Django cache."""
+    cached = get_cached_public_search_state(request)
+    if isinstance(cached, dict):
+        return cached.get("ids") or []
+    return None
+
+
+def get_cached_public_search_state(request) -> dict | None:
+    """Return streamed public search state from Django cache."""
+    cached = cache.get(get_public_search_cache_key(request))
+    return cached if isinstance(cached, dict) else None
+
+
+def iter_url_search_prefixes(query: str):
+    """Yield URL prefixes that can use indexed startswith scans for common search input."""
+    query = query.strip().lower()
+    if not query or any(char.isspace() for char in query):
+        return
+
+    prefixes = []
+
+    def add(prefix: str):
+        if prefix and prefix not in prefixes:
+            prefixes.append(prefix)
+
+    add(query)
+    if "://" in query:
+        parsed = urlsplit(query)
+        if parsed.scheme and parsed.netloc:
+            host = parsed.netloc
+            path = parsed.path or ""
+            if parsed.query:
+                path = f"{path}?{parsed.query}"
+            if host.startswith("www."):
+                add(f"{parsed.scheme}://{host[4:]}{path}")
+            else:
+                add(f"{parsed.scheme}://www.{host}{path}")
+    else:
+        trimmed = query.lstrip("/")
+        for scheme in ("https://", "http://"):
+            add(f"{scheme}{trimmed}")
+            if trimmed.startswith("www."):
+                add(f"{scheme}{trimmed[4:]}")
+            else:
+                add(f"{scheme}www.{trimmed}")
+
+    yield from prefixes
+
+
+def url_prefix_upper_bound(prefix: str) -> str:
+    """Return the exclusive upper bound for an indexed URL prefix range."""
+    if not prefix:
+        return prefix
+    return f"{prefix[:-1]}{chr(ord(prefix[-1]) + 1)}"
+
+
+def iter_url_prefix_search_ids(prefix: str, queryset):
+    """Yield IDs for one URL prefix using the URL index, then apply caller filters."""
+    if not prefix:
+        return
+
+    model = queryset.model
+    db_alias = queryset.db
+    connection = connections[db_alias]
+    table = connection.ops.quote_name(model._meta.db_table)
+    pk_column = connection.ops.quote_name(model._meta.pk.column)
+    url_column = connection.ops.quote_name(model._meta.get_field("url").column)
+    upper_bound = url_prefix_upper_bound(prefix)
+    raw_ids = []
+
+    with connection.cursor() as cursor:
+        cursor.execute(
+            f"""
+            SELECT {pk_column}
+            FROM {table}
+            WHERE {url_column} >= %s AND {url_column} < %s
+            ORDER BY {url_column}
+            LIMIT %s
+            """,
+            [prefix, upper_bound, URL_PREFIX_SEARCH_LIMIT],
+        )
+        raw_ids = [str(row[0]).replace("-", "") for row in cursor.fetchall()]
+
+    if not raw_ids:
+        return
+
+    valid_ids = {str(pk).replace("-", "") for pk in queryset.filter(pk__in=raw_ids).values_list("pk", flat=True)}
+    for snapshot_id in raw_ids:
+        if snapshot_id in valid_ids:
+            yield snapshot_id
+
+
+def iter_meta_search_ids(query, queryset):
     """Yield metadata search matches from a filtered Snapshot queryset."""
     seen = set()
     try:
@@ -54,18 +161,20 @@ def iter_admin_meta_search_ids(query, queryset):
             seen.add(pk)
             yield pk
 
+    for prefix in iter_url_search_prefixes(query):
+        for pk in iter_url_prefix_search_ids(prefix, queryset):
+            if pk in seen:
+                continue
+            seen.add(pk)
+            yield pk
+
     waves = [
-        Q(timestamp__startswith=query) | Q(url__istartswith=query) | Q(title__istartswith=query),
+        Q(timestamp__startswith=query) | Q(title__istartswith=query),
         Q(url__icontains=query),
         Q(title__icontains=query),
         Q(tags__name__icontains=query),
-        Q(notes__icontains=query) | Q(crawl__notes__icontains=query) | Q(crawl__label__icontains=query),
-        Q(crawl__created_by__username=query),
+        Q(notes__icontains=query),
     ]
-    config_wave = crawl_config_values_search_wave(query)
-    if config_wave is not None:
-        waves.append(config_wave)
-
     for wave in waves:
         for pk in queryset.filter(wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500):
             if pk in seen:
@@ -73,38 +182,157 @@ def iter_admin_meta_search_ids(query, queryset):
             seen.add(pk)
             yield pk
 
+    crawl_metadata_wave = Q(crawl__notes__icontains=query) | Q(crawl__label__icontains=query) | Q(crawl__created_by__username=query)
+    if not seen:
+        for pk in queryset.filter(crawl_metadata_wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500):
+            seen.add(pk)
+            yield pk
 
-def iter_admin_backend_search_ids(iterator, queryset):
-    """Yield backend search IDs that still match the filtered queryset."""
+    config_wave = crawl_config_values_search_wave(query)
+    if config_wave is not None and not seen:
+        for pk in queryset.filter(config_wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500):
+            seen.add(pk)
+            yield pk
+
+
+def normalize_search_result_id(snapshot_id) -> str | None:
+    """Return a compact Snapshot ID string from a search provider result."""
+    snapshot_id = str(snapshot_id).strip().lower().replace("-", "")
+    if len(snapshot_id) != 32:
+        return None
+    return snapshot_id
+
+
+def iter_filtered_search_result_ids(iterator, queryset, *, flush_max_delay=0.05):
+    """Yield provider IDs that still match the filtered queryset.
+
+    This is the single intersection/dedupe path used for metadata and every
+    search backend. It flushes by elapsed time so sparse providers stream rows
+    as soon as IDs are found instead of waiting for a fixed batch size.
+    """
     batch = []
     seen = set()
+    queued = set()
+    last_flush_at = 0.0
 
     def flush_batch():
-        valid = {str(pk) for pk in queryset.filter(pk__in=batch).values_list("pk", flat=True)}
-        for snapshot_id in batch:
+        nonlocal batch, queued, last_flush_at
+        if not batch:
+            return
+        batch_ids = batch
+        batch = []
+        queued = set()
+        last_flush_at = time.monotonic()
+        valid = {str(pk).replace("-", "") for pk in queryset.filter(pk__in=batch_ids).values_list("pk", flat=True)}
+        for snapshot_id in batch_ids:
             if snapshot_id in valid and snapshot_id not in seen:
                 seen.add(snapshot_id)
                 yield snapshot_id
 
     for snapshot_id in iterator:
-        snapshot_id = str(snapshot_id).strip().lower().replace("-", "")
-        if len(snapshot_id) != 32:
+        snapshot_id = normalize_search_result_id(snapshot_id)
+        if not snapshot_id or snapshot_id in seen or snapshot_id in queued:
             continue
         batch.append(snapshot_id)
-        if len(batch) >= 200:
+        queued.add(snapshot_id)
+        if not seen or time.monotonic() - last_flush_at >= flush_max_delay:
             yield from flush_batch()
-            batch = []
     if batch:
         yield from flush_batch()
 
 
+def iter_search_result_ids(query, base_queryset, *, search_mode, config):
+    """Yield filtered Snapshot IDs from the selected search provider."""
+    search_mode_base = get_search_mode_base(search_mode, config=config)
+    provider = (
+        iter_meta_search_ids(query, base_queryset)
+        if search_mode_base == "meta"
+        else iter_query_search_ids(query, search_mode=search_mode, config=config)
+    )
+    yield from iter_filtered_search_result_ids(provider, base_queryset)
+
+
+def snapshot_search_stream_response(query, base_queryset, *, search_mode, config, cache_key, thread_name):
+    """Stream Snapshot search progress and cache matching IDs for a list view."""
+    if not query:
+        return StreamingHttpResponse((), content_type="text/plain")
+
+    async def snapshot_ids():
+        seen = set()
+        ids = []
+        last_sent = 0
+        last_sent_at = time.monotonic()
+        stream_max_delay = 0.05
+        stream_padding = " " * 4096
+        cache.set(cache_key, {"ids": [], "done": False}, SEARCH_RESULT_CACHE_TTL)
+        queue = Queue(maxsize=8)
+        stop_event = threading.Event()
+
+        def emit(item):
+            while not stop_event.is_set():
+                try:
+                    queue.put(item, timeout=0.1)
+                    return
+                except Full:
+                    continue
+
+        def publish_count(done=False):
+            nonlocal last_sent, last_sent_at
+            cache.set(cache_key, {"ids": list(ids), "done": done}, SEARCH_RESULT_CACHE_TTL)
+            last_sent = len(ids)
+            last_sent_at = time.monotonic()
+            emit(f"{last_sent}{stream_padding}\n")
+
+        def run_search():
+            iterator = None
+            try:
+                iterator = iter_search_result_ids(query, base_queryset, search_mode=search_mode, config=config)
+                for snapshot_id in iterator:
+                    if stop_event.is_set():
+                        break
+                    snapshot_id = normalize_search_result_id(snapshot_id)
+                    if not snapshot_id or snapshot_id in seen:
+                        continue
+                    seen.add(snapshot_id)
+                    ids.append(snapshot_id)
+                    if len(ids) == 1 or time.monotonic() - last_sent_at >= stream_max_delay:
+                        publish_count()
+                if not stop_event.is_set() and len(ids) != last_sent:
+                    publish_count(done=True)
+            except BaseException as err:
+                emit(err)
+            finally:
+                if iterator is not None:
+                    try:
+                        iterator.close()
+                    except AttributeError:
+                        pass
+                cache.set(cache_key, {"ids": list(ids), "done": True}, SEARCH_RESULT_CACHE_TTL)
+                emit(None)
+
+        threading.Thread(target=run_search, name=thread_name, daemon=True).start()
+        yield f"0{stream_padding}\n"
+        try:
+            while True:
+                item = await asyncio.to_thread(queue.get)
+                if item is None:
+                    break
+                if isinstance(item, BaseException):
+                    raise item
+                yield item
+        finally:
+            stop_event.set()
+
+    response = StreamingHttpResponse(snapshot_ids(), content_type="text/plain")
+    response["X-Accel-Buffering"] = "no"
+    return response
+
+
 def admin_snapshot_search_stream_view(model_admin, request):
     """Stream admin Snapshot search progress and cache matching IDs."""
     query = (request.GET.get("q") or "").strip()
     config = request.archivebox_config
     search_mode = get_search_mode(request.GET.get("search_mode"), config=config)
-    if not query:
-        return StreamingHttpResponse((), content_type="text/plain")
 
     search_url = request.GET.get("search_url") or request.get_full_path()
     target_url = urlsplit(search_url)
@@ -127,77 +355,36 @@ def admin_snapshot_search_stream_view(model_admin, request):
     finally:
         model_admin.request = current_request
 
-    async def snapshot_ids():
-        seen = set()
-        ids = []
-        last_sent = 0
-        stream_batch_size = 100
-        stream_padding = " " * 4096
-        cache_key = get_admin_search_cache_key(request, search_url)
-        cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-        yield f"0{stream_padding}\n"
-        queue = Queue(maxsize=8)
-        stop_event = threading.Event()
+    return snapshot_search_stream_response(
+        query,
+        base_queryset,
+        search_mode=search_mode,
+        config=config,
+        cache_key=get_admin_search_cache_key(request, search_url),
+        thread_name="admin-snapshot-search-stream",
+    )
 
-        def emit(item):
-            while not stop_event.is_set():
-                try:
-                    queue.put(item, timeout=0.1)
-                    return
-                except Full:
-                    continue
 
-        def run_search():
-            nonlocal last_sent
-            iterator = None
-            try:
-                search_mode_base = get_search_mode_base(search_mode, config=config)
-                iterator = (
-                    iter_admin_meta_search_ids(query, base_queryset)
-                    if search_mode_base == "meta"
-                    else iter_admin_backend_search_ids(
-                        iter_query_search_ids(query, search_mode=search_mode, config=config),
-                        base_queryset,
-                    )
-                )
-                for snapshot_id in iterator:
-                    if stop_event.is_set():
-                        break
-                    snapshot_id = str(snapshot_id).strip().lower().replace("-", "")
-                    if len(snapshot_id) != 32 or snapshot_id in seen:
-                        continue
-                    seen.add(snapshot_id)
-                    ids.append(snapshot_id)
-                    if len(ids) - last_sent >= stream_batch_size:
-                        cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-                        last_sent = len(ids)
-                        emit(f"{last_sent}{stream_padding}\n")
-                if not stop_event.is_set() and len(ids) != last_sent:
-                    cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-                    emit(f"{len(ids)}{stream_padding}\n")
-            except BaseException as err:
-                emit(err)
-            finally:
-                if iterator is not None:
-                    try:
-                        iterator.close()
-                    except AttributeError:
-                        pass
-                cache.set(cache_key, {"ids": ids, "done": True}, SEARCH_RESULT_CACHE_TTL)
-                emit(None)
+def public_snapshot_search_stream_view(request):
+    """Stream public Snapshot search progress and cache matching IDs."""
+    from archivebox.config.common import get_request_config
+    from archivebox.core.models import Snapshot
+    from archivebox.core.permissions import public_snapshots_queryset
 
-        threading.Thread(target=run_search, name="admin-snapshot-search-stream", daemon=True).start()
-        try:
-            while True:
-                item = await asyncio.to_thread(queue.get)
-                if item is None:
-                    break
-                if isinstance(item, BaseException):
-                    raise item
-                yield item
-        finally:
-            stop_event.set()
+    config = getattr(request, "archivebox_config", None) or get_request_config(request, resolve_plugins=False)
+    if not request.user.is_authenticated and not config.PUBLIC_INDEX:
+        return HttpResponseForbidden("Public index is disabled")
 
-    response = StreamingHttpResponse(snapshot_ids(), content_type="text/plain")
-    response["X-Accel-Buffering"] = "no"
-    return response
+    query = (request.GET.get("q") or "").strip()
+    search_mode = get_search_mode(request.GET.get("search_mode"), config=config)
+    search_url = request.GET.get("search_url") or request.get_full_path()
+    base_queryset = public_snapshots_queryset(Snapshot.objects.all())
+
+    return snapshot_search_stream_response(
+        query,
+        base_queryset,
+        search_mode=search_mode,
+        config=config,
+        cache_key=get_public_search_cache_key(request, search_url),
+        thread_name="public-snapshot-search-stream",
+    )

archivebox/services/archive_result_service.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import inspect
 import json
 import os
 import sys
@@ -25,7 +26,7 @@ from .process_service import parse_event_datetime
 
 def _perf_trace(label):
     def decorator(func):
-        if asyncio.iscoroutinefunction(func):
+        if inspect.iscoroutinefunction(func):
 
             @wraps(func)
             async def async_wrapper(*args, **kwargs):
@@ -74,7 +75,6 @@ class ModelDumpable(Protocol):
     def model_dump(self) -> dict[str, Any]: ...
 
 
-@_perf_trace("archivebox.ArchiveResultService._collect_output_metadata")
 def _collect_output_metadata(plugin_dir: Path) -> tuple[dict[str, dict], int, str]:
     exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid"}
     output_files: dict[str, dict] = {}
@@ -187,7 +187,6 @@ def _summarize_output_files(output_files: dict[str, dict]) -> tuple[int, str]:
     return total_size, output_mimetypes
 
 
-@_perf_trace("archivebox.ArchiveResultService._resolve_output_metadata")
 def _resolve_output_metadata(raw_output_files: Any, plugin_dir: Path) -> tuple[dict[str, dict], int, str]:
     normalized_output_files = _normalize_output_files(raw_output_files)
     if normalized_output_files and _has_structured_output_metadata(normalized_output_files):
@@ -242,6 +241,10 @@ def _has_content_files(output_files: Any) -> bool:
     return any(Path(path).suffix not in {".log", ".pid", ".sh"} for path in _normalize_output_files(output_files))
 
 
+def _is_signal_interrupted_exit(exit_code: int) -> bool:
+    return exit_code < 0 or (exit_code >= 128 and exit_code != PROCESS_EXIT_SKIPPED)
+
+
 def _iter_archiveresult_records(stdout: str) -> list[dict]:
     records: list[dict] = []
     for raw_line in stdout.splitlines():
@@ -257,6 +260,119 @@ def _iter_archiveresult_records(stdout: str) -> list[dict]:
     return records
 
 
+@_perf_trace("archivebox.ArchiveResultService._save_archiveresult_event_sync")
+def _save_archiveresult_event_to_db(
+    event: ArchiveResultEvent,
+    process_started: ProcessStartedEvent | None,
+) -> None:
+    """Project one ArchiveResultEvent with a single thread-sensitive ORM hop.
+
+    Django's async ORM still delegates each query to sync Django work. The hot
+    search/index maintenance path was paying that handoff separately for
+    Snapshot lookup, Process lookup, ArchiveResult lookup, update, and title
+    checks. Keep the public ArchiveResultEvent path intact, but run the DB
+    projection as one short synchronous block so SQLite sees the same indexed
+    reads/writes without per-query asyncio/threadpool churn.
+    """
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.machine.models import Process
+
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.snapshot_lookup"):
+        snapshot = Snapshot.objects.filter(id=event.snapshot_id).select_related("crawl", "crawl__created_by").first()
+    if snapshot is None:
+        return
+
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.plugin_dir"):
+        plugin_dir = (
+            Path(process_started.output_dir)
+            if process_started is not None and process_started.output_dir
+            else Path(snapshot.output_dir) / event.plugin
+        )
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.resolve_output_metadata"):
+        output_files, output_size, output_mimetypes = _resolve_output_metadata(event.output_files, plugin_dir)
+
+    process = None
+    if process_started is not None:
+        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.process_lookup"):
+            started_at = parse_event_datetime(process_started.start_ts)
+            if started_at is None:
+                raise ValueError("ProcessStartedEvent.start_ts is required")
+            process_query = Process.objects.filter(
+                pwd=process_started.output_dir,
+                cmd=[process_started.hook_path, *process_started.hook_args],
+                started_at=started_at,
+            )
+            if process_started.pid:
+                process_query = process_query.filter(pid=process_started.pid)
+            process = process_query.order_by("-modified_at").first()
+
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.prepare_defaults"):
+        start_ts = parse_event_datetime(event.start_ts)
+        end_ts = parse_event_datetime(event.end_ts) or timezone.now()
+        defaults = {
+            "status": _normalize_status(event.status),
+            "output_str": event.output_str,
+            "output_json": event.output_json,
+            "output_files": output_files,
+            "output_size": output_size,
+            "output_mimetypes": output_mimetypes,
+            "start_ts": start_ts or timezone.now(),
+            "end_ts": end_ts,
+        }
+        if process is not None:
+            defaults["process_id"] = process.id
+        if event.error:
+            defaults["notes"] = event.error
+
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_lookup"):
+        result = ArchiveResult.objects.filter(
+            snapshot=snapshot,
+            plugin=event.plugin,
+            hook_name=event.hook_name,
+        ).first()
+    if result is None:
+        try:
+            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_create"):
+                result = ArchiveResult.objects.create(
+                    snapshot=snapshot,
+                    plugin=event.plugin,
+                    hook_name=event.hook_name,
+                    **defaults,
+                )
+        except IntegrityError:
+            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_get_after_integrity"):
+                result = ArchiveResult.objects.get(
+                    snapshot=snapshot,
+                    plugin=event.plugin,
+                    hook_name=event.hook_name,
+                )
+
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.diff_fields"):
+        update_fields = []
+        for field, value in defaults.items():
+            if result.__dict__[field] != value:
+                setattr(result, field, value)
+                update_fields.append(field)
+    if update_fields:
+        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_update"):
+            result.save(update_fields=[*update_fields, "modified_at"])
+
+    if result.status == ArchiveResult.StatusChoices.QUEUED:
+        # ArchiveResult has no retry_at column. If a shutdown/takeover projects
+        # a killed hook back to QUEUED, wake the parent Snapshot/Crawl so the
+        # next runner retries that exact hook instead of waiting on a stale
+        # active-state lease.
+        snapshot.update_and_requeue(retry_at=timezone.now())
+
+    if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS):
+        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.title_update"):
+            title_output_str = result.output_str if result.status == ArchiveResult.StatusChoices.SUCCEEDED else ""
+            next_title = _extract_snapshot_title(str(plugin_dir.parent), event.plugin, title_output_str, snapshot_url=snapshot.url)
+            if next_title and _should_update_snapshot_title(snapshot.title or "", next_title, snapshot_url=snapshot.url):
+                snapshot.title = next_title
+                snapshot.save(update_fields=["title", "modified_at"])
+
+
 class ArchiveResultService(BaseService):
     LISTENS_TO = [ArchiveResultEvent, ProcessCompletedEvent]
     EMITS = []
@@ -270,13 +386,6 @@ class ArchiveResultService(BaseService):
 
     @_perf_trace("archivebox.ArchiveResultService.on_ArchiveResultEvent__save_to_db")
     async def on_ArchiveResultEvent__save_to_db(self, event: ArchiveResultEvent) -> None:
-        from archivebox.core.models import ArchiveResult, Snapshot
-        from archivebox.machine.models import Process
-
-        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.snapshot_lookup"):
-            snapshot = await Snapshot.objects.filter(id=event.snapshot_id).select_related("crawl", "crawl__created_by").afirst()
-        if snapshot is None:
-            return
         with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.find_process_started"):
             process_started = await self.bus.find(
                 ProcessStartedEvent,
@@ -284,94 +393,11 @@ class ArchiveResultService(BaseService):
                 future=False,
                 where=lambda candidate: self.bus.event_is_child_of(event, candidate),
             )
-        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.plugin_dir"):
-            plugin_dir = (
-                Path(process_started.output_dir)
-                if process_started is not None and process_started.output_dir
-                else Path(snapshot.output_dir) / event.plugin
-            )
-        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.resolve_output_metadata"):
-            output_files, output_size, output_mimetypes = await sync_to_async(_resolve_output_metadata)(event.output_files, plugin_dir)
-        process = None
-        if process_started is not None:
-            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.process_lookup"):
-                started_at = parse_event_datetime(process_started.start_ts)
-                if started_at is None:
-                    raise ValueError("ProcessStartedEvent.start_ts is required")
-                process_query = Process.objects.filter(
-                    pwd=process_started.output_dir,
-                    cmd=[process_started.hook_path, *process_started.hook_args],
-                    started_at=started_at,
-                )
-                if process_started.pid:
-                    process_query = process_query.filter(pid=process_started.pid)
-                process = await process_query.order_by("-modified_at").afirst()
 
-        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.prepare_defaults"):
-            start_ts = parse_event_datetime(event.start_ts)
-            end_ts = parse_event_datetime(event.end_ts) or timezone.now()
-            defaults = {
-                "status": _normalize_status(event.status),
-                "output_str": event.output_str,
-                "output_json": event.output_json,
-                "output_files": output_files,
-                "output_size": output_size,
-                "output_mimetypes": output_mimetypes,
-                "start_ts": start_ts or timezone.now(),
-                "end_ts": end_ts,
-            }
-            if process is not None:
-                defaults["process_id"] = process.id
-            if event.error:
-                defaults["notes"] = event.error
-
-        key = (str(snapshot.id), event.plugin, event.hook_name)
+        key = (str(event.snapshot_id), event.plugin, event.hook_name)
         lock = self._save_locks.setdefault(key, asyncio.Lock())
         async with lock:
-            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_lookup"):
-                # Avoid update_or_create(): Django wraps it in transaction.atomic().
-                # On SQLite a lock retry inside that read-then-write transaction can
-                # keep the read transaction open and block the writer that would have
-                # released the lock. Use autocommit-sized writes instead.
-                result = await ArchiveResult.objects.filter(
-                    snapshot=snapshot,
-                    plugin=event.plugin,
-                    hook_name=event.hook_name,
-                ).afirst()
-            if result is None:
-                try:
-                    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_create"):
-                        result = await ArchiveResult.objects.acreate(
-                            snapshot=snapshot,
-                            plugin=event.plugin,
-                            hook_name=event.hook_name,
-                            **defaults,
-                        )
-                except IntegrityError:
-                    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_get_after_integrity"):
-                        result = await ArchiveResult.objects.aget(
-                            snapshot=snapshot,
-                            plugin=event.plugin,
-                            hook_name=event.hook_name,
-                        )
-
-            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.diff_fields"):
-                update_fields = []
-                for field, value in defaults.items():
-                    if result.__dict__[field] != value:
-                        setattr(result, field, value)
-                        update_fields.append(field)
-            if update_fields:
-                with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_update"):
-                    await result.asave(update_fields=[*update_fields, "modified_at"])
-
-        if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS):
-            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.title_update"):
-                title_output_str = result.output_str if result.status == ArchiveResult.StatusChoices.SUCCEEDED else ""
-                next_title = _extract_snapshot_title(str(plugin_dir.parent), event.plugin, title_output_str, snapshot_url=snapshot.url)
-                if next_title and _should_update_snapshot_title(snapshot.title or "", next_title, snapshot_url=snapshot.url):
-                    snapshot.title = next_title
-                    await snapshot.asave(update_fields=["title", "modified_at"])
+            await sync_to_async(_save_archiveresult_event_to_db, thread_sensitive=True)(event, process_started)
 
     @_perf_trace("archivebox.ArchiveResultService.on_ProcessCompletedEvent__save_to_db")
     async def on_ProcessCompletedEvent__save_to_db(self, event: ProcessCompletedEvent) -> None:
@@ -418,14 +444,24 @@ class ArchiveResultService(BaseService):
                     ).now()
             return
 
-        process_failed = event.exit_code not in (0, PROCESS_EXIT_SKIPPED)
+        # TODO: consider moving this fallback derivation into abx-dl itself.
+        # First try both patterns: if the whole abx-dl process crashes, restarting
+        # the snapshot may be enough, but don't guess before validating it.
+        process_interrupted = _is_signal_interrupted_exit(event.exit_code)
+        process_failed = event.exit_code not in (0, PROCESS_EXIT_SKIPPED) and not process_interrupted
         with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_fallback"):
             await event.emit(
                 ArchiveResultEvent(
                     snapshot_id=snapshot_event.snapshot_id,
                     plugin=event.plugin_name,
                     hook_name=event.hook_name,
-                    status="failed" if process_failed else ("succeeded" if _has_content_files(event.output_files) else "skipped"),
+                    status=(
+                        "queued"
+                        if process_interrupted
+                        else "failed"
+                        if process_failed
+                        else ("succeeded" if _has_content_files(event.output_files) else "noresult")
+                    ),
                     output_str=event.stderr if process_failed else "",
                     output_files=event.output_files,
                     start_ts=event.start_ts,

archivebox/services/binary_service.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import json
 from collections.abc import Mapping
 from pathlib import Path
@@ -15,21 +16,6 @@ from abxbus import BaseEvent, EventBus
 from abx_dl.services.base import BaseService
 
 
-_LIB_DIR_MANAGED_PROVIDERS = {
-    "bash",
-    "cargo",
-    "deno",
-    "gem",
-    "goget",
-    "chromewebstore",
-    "nix",
-    "npm",
-    "pip",
-    "puppeteer",
-    "uv",
-}
-
-
 class ArchiveBoxDBBinaryCacheBackend:
     """ArchiveBox machine.Binary projection backend for abxpkg BinaryCacheService."""
 
@@ -42,11 +28,62 @@ class ArchiveBoxDBBinaryCacheBackend:
         if not binary_name:
             return None
 
-        existing = await Binary.objects.filter(machine=machine, name=binary_name).afirst()
         persisted_overrides = _persisted_overrides_for_request(request)
         native_overrides = request.overrides or {}
-        cache_invalidated = False
-        if existing and existing.status == Binary.StatusChoices.INSTALLED:
+        requested_provider_names = _provider_names(request.binproviders)
+        await sync_to_async(get_config, thread_sensitive=True)()
+
+        installed_qs = (
+            Binary.objects.filter(machine=machine, name=binary_name, status=Binary.StatusChoices.INSTALLED)
+            .exclude(abspath="")
+            .exclude(abspath__isnull=True)
+            .order_by("-modified_at")
+        )
+        async for installed in installed_qs:
+            installed_path = Path(installed.abspath).expanduser().resolve(strict=False)
+            if not await sync_to_async(installed_path.exists, thread_sensitive=True)():
+                await _mark_binary_queued(installed)
+                continue
+            if persisted_overrides and installed.overrides != persisted_overrides:
+                await _mark_binary_queued(installed)
+                continue
+
+            provider_name = (installed.binprovider or installed.binproviders.split(",", 1)[0]).strip()
+            if provider_name and provider_name not in requested_provider_names:
+                await _mark_binary_queued(installed)
+                continue
+
+            provider = _provider_for_name(provider_name, installed.name, native_overrides)
+            if await sync_to_async(_cached_provider_path_is_stale, thread_sensitive=True)(installed_path, provider, installed.name):
+                await _mark_binary_queued(installed)
+                continue
+
+            binary_env = BinProvider.build_exec_env(providers=[provider], base_env={}) if provider is not None else {}
+            provider_names = _provider_names(installed.binproviders or request.binproviders or "env")
+            return AbxBinary.model_validate(
+                {
+                    "name": request.name,
+                    "description": request.description,
+                    "binproviders": _providers_for_names(provider_names),
+                    "overrides": native_overrides,
+                    "loaded_binprovider": provider,
+                    "loaded_abspath": installed.abspath,
+                    "loaded_version": installed.version or None,
+                    "loaded_sha256": installed.sha256 or None,
+                    "env": binary_env,
+                },
+            )
+
+        existing = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst()
+        if existing is None:
+            await Binary.objects.acreate(
+                machine=machine,
+                name=binary_name,
+                binproviders=_binproviders_to_str(request.binproviders),
+                overrides=persisted_overrides,
+                status=Binary.StatusChoices.QUEUED,
+            )
+        else:
             changed = False
             requested_binproviders = _binproviders_to_str(request.binproviders)
             if requested_binproviders and existing.binproviders != requested_binproviders:
@@ -55,75 +92,15 @@ class ArchiveBoxDBBinaryCacheBackend:
             if persisted_overrides and existing.overrides != persisted_overrides:
                 existing.overrides = persisted_overrides
                 changed = True
-            if changed:
+            if existing.status != Binary.StatusChoices.QUEUED:
                 existing.status = Binary.StatusChoices.QUEUED
                 existing.retry_at = None
-                cache_invalidated = True
+                changed = True
+            if changed:
                 await existing.asave(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
-        elif existing is None:
-            await Binary.objects.acreate(
-                machine=machine,
-                name=binary_name,
-                binproviders=_binproviders_to_str(request.binproviders),
-                overrides=persisted_overrides,
-                status=Binary.StatusChoices.QUEUED,
-            )
-
-        installed = None
-        if not cache_invalidated:
-            installed = (
-                await Binary.objects.filter(machine=machine, name=binary_name, status=Binary.StatusChoices.INSTALLED)
-                .exclude(abspath="")
-                .exclude(abspath__isnull=True)
-                .order_by("-modified_at")
-                .afirst()
-            )
-            if installed is not None and not await sync_to_async(Path(installed.abspath).expanduser().exists, thread_sensitive=True)():
-                installed.status = Binary.StatusChoices.QUEUED
-                installed.retry_at = None
-                await installed.asave(update_fields=["status", "retry_at", "modified_at"])
-                installed = None
-            if installed is not None and persisted_overrides and installed.overrides != persisted_overrides:
-                installed.status = Binary.StatusChoices.QUEUED
-                installed.retry_at = None
-                await installed.asave(update_fields=["status", "retry_at", "modified_at"])
-                installed = None
-        if installed is None:
-            return None
-
-        installed_path = Path(installed.abspath).expanduser().resolve(strict=False)
-        active_lib_dir = (
-            Path(str((await sync_to_async(get_config, thread_sensitive=True)()).get("LIB_DIR", ""))).expanduser().resolve(strict=False)
-        )
-        provider_name = (installed.binprovider or installed.binproviders.split(",", 1)[0]).strip()
-        if active_lib_dir and provider_name in _LIB_DIR_MANAGED_PROVIDERS:
-            try:
-                installed_path.relative_to(active_lib_dir)
-            except ValueError:
-                installed.status = Binary.StatusChoices.QUEUED
-                installed.retry_at = None
-                await installed.asave(update_fields=["status", "retry_at", "modified_at"])
-                return None
-
-        provider = _provider_for_name(provider_name, installed.name, native_overrides)
-        binary_env = BinProvider.build_exec_env(providers=[provider], base_env={}) if provider is not None else {}
-        provider_names = _provider_names(installed.binproviders or request.binproviders or "env")
-        return AbxBinary.model_validate(
-            {
-                "name": request.name,
-                "description": request.description,
-                "binproviders": _providers_for_names(provider_names),
-                "overrides": native_overrides,
-                "loaded_binprovider": provider,
-                "loaded_abspath": installed.abspath,
-                "loaded_version": installed.version or None,
-                "loaded_sha256": installed.sha256 or None,
-                "env": binary_env,
-            },
-        )
+        return None
 
     async def set(self, request: BinaryRequestEvent | None, binary: AbxBinary) -> None:
-        from archivebox.config.common import get_config
         from archivebox.machine.models import Binary, Machine, _canonical_binary_name
 
         machine = await sync_to_async(Machine.current, thread_sensitive=True)()
@@ -159,8 +136,6 @@ class ArchiveBoxDBBinaryCacheBackend:
         await existing.asave(
             update_fields=["abspath", "version", "sha256", "binproviders", "binprovider", "overrides", "status", "retry_at", "modified_at"],
         )
-        lib_bin_dir = await sync_to_async(lambda: get_config().LIB_BIN_DIR, thread_sensitive=True)()
-        await sync_to_async(existing.symlink_to_lib_bin_after_commit, thread_sensitive=True)(lib_bin_dir)
 
     async def invalidate(self, request: BinaryRequestEvent, binary: AbxBinary, reason: str) -> None:
         from archivebox.machine.models import Binary, Machine, _canonical_binary_name
@@ -186,38 +161,110 @@ class ArchiveBoxDBBinaryCacheBackend:
 class ArchiveBoxBinaryService(BaseService):
     """Preserve ArchiveBox's legacy Binary Process rows around abxpkg requests."""
 
-    LISTENS_TO = [BinaryRequestEvent]
+    LISTENS_TO = [BinaryRequestEvent, BinaryEvent]
     EMITS: list[type[BaseEvent]] = []
 
     def __init__(self, bus: EventBus):
         super().__init__(bus)
+        self.process_ids_by_request_id: dict[str, str] = {}
+        self._missing_finalize_tasks: set[asyncio.Task] = set()
         self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent__project_process)
+        self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent__schedule_missing_finalize)
+        self.bus.on(BinaryEvent, self.on_BinaryEvent__finalize_process)
 
     async def on_BinaryRequestEvent__project_process(self, request: BinaryRequestEvent) -> None:
-        from archivebox.machine.models import Binary, Machine, Process, _canonical_binary_name
-        from archivebox.services.process_service import current_network_interface_with_machine
+        from archivebox.machine.models import Machine, Process, _canonical_binary_name
 
         machine = await sync_to_async(Machine.current, thread_sensitive=True)()
         binary_name = _canonical_binary_name(request.name)
         if not binary_name:
             return
-        binary = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst()
-        if binary is None:
-            return
+        binary = await self._get_or_create_binary(machine, binary_name, request)
+        started_at = timezone.now()
+        output_dir = self._process_output_dir(binary, request)
+        await sync_to_async(output_dir.mkdir, thread_sensitive=True)(parents=True, exist_ok=True)
+        process = await Process.objects.acreate(
+            machine=machine,
+            iface=None,
+            process_type=Process.TypeChoices.BINARY,
+            worker_type="",
+            pwd=str(output_dir),
+            cmd=self._process_cmd(request),
+            env={},
+            timeout=int(request.event_timeout or request.install_timeout or 600),
+            pid=None,
+            url=None,
+            started_at=started_at,
+            ended_at=None,
+            stdout="",
+            stderr="",
+            exit_code=None,
+            status=Process.StatusChoices.RUNNING,
+            retry_at=None,
+            binary=binary,
+        )
+        self.process_ids_by_request_id[request.event_id] = str(process.id)
 
-        binary_event = await self.bus.find(
-            BinaryEvent,
-            child_of=request,
+    async def on_BinaryEvent__finalize_process(self, event: BinaryEvent) -> None:
+        from archivebox.machine.models import Binary, Process, _canonical_binary_name
+
+        request = await self.bus.find(
+            BinaryRequestEvent,
             past=True,
             future=False,
-            name=request.name,
-            where=lambda candidate: bool(candidate.abspath),
+            where=lambda candidate: self.bus.event_is_child_of(event, candidate),
         )
-        iface = await sync_to_async(current_network_interface_with_machine, thread_sensitive=True)()
-        now = timezone.now()
-        success = isinstance(binary_event, BinaryEvent)
-        output_dir = self._process_output_dir(binary, request)
-        output_dir.mkdir(parents=True, exist_ok=True)
+        request = request if isinstance(request, BinaryRequestEvent) else None
+        process_id = self.process_ids_by_request_id.pop(request.event_id, "") if request is not None else ""
+        if not process_id:
+            return
+        process = await Process.objects.filter(id=process_id).select_related("binary").afirst()
+        if process is None:
+            return
+        binary_name = _canonical_binary_name(event.name)
+        binary = process.binary
+        if binary is not None and binary_name:
+            binary.abspath = event.abspath
+            if event.version:
+                binary.version = str(event.version)
+            if event.sha256:
+                binary.sha256 = str(event.sha256)
+            binary.binproviders = event.binproviders or binary.binproviders
+            binary.binprovider = event.binprovider or binary.binprovider
+            binary.status = Binary.StatusChoices.INSTALLED
+            binary.retry_at = None
+            await binary.asave(
+                update_fields=["abspath", "version", "sha256", "binproviders", "binprovider", "status", "retry_at", "modified_at"],
+            )
+        process.ended_at = timezone.now()
+        process.stdout = json.dumps(self._binary_event_json(event, binary)) + "\n"
+        process.stderr = ""
+        process.exit_code = 0
+        process.status = Process.StatusChoices.EXITED
+        await process.asave(update_fields=["ended_at", "stdout", "stderr", "exit_code", "status", "modified_at"])
+        if binary is not None:
+            await sync_to_async(self._write_binary_index, thread_sensitive=True)(binary, process, Path(process.pwd))
+
+    async def _get_or_create_binary(self, machine, binary_name: str, request: BinaryRequestEvent):
+        from archivebox.machine.models import Binary
+
+        binary_id = str(request.extra_context.get("binary_id") or "")
+        if binary_id:
+            binary = await Binary.objects.filter(id=binary_id).afirst()
+            if binary is not None:
+                return binary
+        binary = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst()
+        if binary is not None:
+            return binary
+        return await Binary.objects.acreate(
+            machine=machine,
+            name=binary_name,
+            binproviders=_binproviders_to_str(request.binproviders),
+            overrides=_persisted_overrides_for_request(request),
+            status=Binary.StatusChoices.QUEUED,
+        )
+
+    def _process_cmd(self, request: BinaryRequestEvent) -> list[str]:
         cmd = [
             "abxpkg",
             "install",
@@ -226,29 +273,70 @@ class ArchiveBoxBinaryService(BaseService):
         ]
         if request.overrides:
             cmd.append(f"--overrides={json.dumps(request.overrides, sort_keys=True)}")
-        stdout = json.dumps(binary.to_json()) + "\n" if success else ""
-        stderr = "" if success else f"Binary request did not resolve: {request.name}"
-        process = await Process.objects.acreate(
-            machine=iface.machine,
-            iface=iface,
-            process_type=Process.TypeChoices.BINARY,
-            worker_type="",
-            pwd=str(output_dir),
-            cmd=cmd,
-            env={},
-            timeout=int(request.event_timeout or request.install_timeout or 600),
-            pid=None,
-            url=None,
-            started_at=now,
-            ended_at=now,
-            stdout=stdout,
-            stderr=stderr,
-            exit_code=0 if success else 1,
-            status=Process.StatusChoices.EXITED,
-            retry_at=None,
-            binary=binary,
+        return cmd
+
+    def _binary_event_json(self, event: BinaryEvent, binary) -> dict[str, Any]:
+        if binary is not None:
+            data = binary.to_json()
+        else:
+            data = {"type": "Binary", "name": event.name}
+        data.update(
+            {
+                "type": "Binary",
+                "name": event.name,
+                "binproviders": event.binproviders,
+                "binprovider": event.binprovider,
+                "abspath": event.abspath,
+                "version": str(event.version or ""),
+                "sha256": event.sha256 or "",
+                "status": "installed",
+            },
         )
-        self._write_binary_index(binary, process, output_dir)
+        return data
+
+    async def _finalize_missing_process(self, request: BinaryRequestEvent) -> None:
+        from archivebox.machine.models import Process
+
+        process_id = self.process_ids_by_request_id.pop(request.event_id, "")
+        if not process_id:
+            return
+        process = await Process.objects.filter(id=process_id).afirst()
+        if process is None or process.status == Process.StatusChoices.EXITED:
+            return
+        process.ended_at = timezone.now()
+        process.stderr = f"Binary request did not resolve: {request.name}"
+        process.exit_code = 1
+        process.status = Process.StatusChoices.EXITED
+        await process.asave(update_fields=["ended_at", "stderr", "exit_code", "status", "modified_at"])
+
+    async def _finalize_request_when_done(self, request: BinaryRequestEvent) -> None:
+        try:
+            await request.wait(timeout=request.event_timeout)
+        except TimeoutError:
+            await self._finalize_missing_process(request)
+            return
+        binary_event = await self.bus.find(
+            BinaryEvent,
+            child_of=request,
+            past=True,
+            future=False,
+            name=request.name,
+            where=lambda candidate: bool(candidate.abspath),
+        )
+        if not isinstance(binary_event, BinaryEvent):
+            await self._finalize_missing_process(request)
+
+    def _schedule_missing_finalize(self, request: BinaryRequestEvent) -> None:
+        task = asyncio.create_task(self._finalize_request_when_done(request))
+        self._missing_finalize_tasks.add(task)
+        task.add_done_callback(lambda done: self._missing_finalize_tasks.discard(done) or (None if done.cancelled() else done.exception()))
+
+    async def flush_missing_finalizers(self) -> None:
+        if self._missing_finalize_tasks:
+            await asyncio.gather(*tuple(self._missing_finalize_tasks), return_exceptions=False)
+
+    async def on_BinaryRequestEvent__schedule_missing_finalize(self, request: BinaryRequestEvent) -> None:
+        self._schedule_missing_finalize(request)
 
     def _process_output_dir(self, binary, request: BinaryRequestEvent) -> Path:
         raw_output_dir = str(request.extra_context.get("output_dir") or "").strip()
@@ -307,6 +395,25 @@ def _provider_for_name(provider_name: str, binary_name: str, overrides: dict[str
     return provider
 
 
+async def _mark_binary_queued(binary) -> None:
+    from archivebox.machine.models import Binary
+
+    if binary.status == Binary.StatusChoices.QUEUED:
+        return
+    binary.status = Binary.StatusChoices.QUEUED
+    binary.retry_at = None
+    await binary.asave(update_fields=["status", "retry_at", "modified_at"])
+
+
+def _cached_provider_path_is_stale(installed_path: Path, provider: BinProvider | None, binary_name: str) -> bool:
+    if provider is None:
+        return False
+    current_abspath = provider.get_abspath(binary_name, quiet=True, no_cache=True)
+    if not current_abspath:
+        return True
+    return Path(current_abspath).expanduser().resolve(strict=False) != installed_path
+
+
 def _persisted_overrides_for_request(request: BinaryRequestEvent | None) -> dict[str, Any]:
     if request is None:
         return {}

archivebox/services/machine_service.py

@@ -1,9 +1,5 @@
 from __future__ import annotations
 
-import os
-import sys
-import time
-from contextlib import contextmanager
 from typing import Any
 
 from asgiref.sync import sync_to_async
@@ -11,21 +7,6 @@ from asgiref.sync import sync_to_async
 from abx_dl.events import MachineEvent
 from abx_dl.services.base import BaseService
 
-_BINARY_EVENT_ALLOWED_KEYS = frozenset({"ABX_INSTALL_CACHE"})
-
-
-@contextmanager
-def _perf_span(label: str):
-    if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
-        yield
-        return
-    started_at = time.perf_counter()
-    try:
-        yield
-    finally:
-        elapsed_ms = (time.perf_counter() - started_at) * 1000
-        print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
-
 
 def _is_binary_event_key(key: str) -> bool:
     """``MachineEvent`` projector only ever writes binary-related state.
@@ -37,7 +18,7 @@ def _is_binary_event_key(key: str) -> bool:
     mirror is a security boundary), so the projector strips anything that
     isn't a binary path or the binary install cache.
     """
-    if key in _BINARY_EVENT_ALLOWED_KEYS:
+    if key.startswith("ABX_") and key.endswith("CACHE"):
         return True
     return key.endswith("_BINARY")
 
@@ -62,29 +43,25 @@ class MachineService(BaseService):
         if event.config_type != "derived":
             return
 
-        with _perf_span("archivebox.MachineService.on_MachineEvent.machine_current"):
-            machine = await sync_to_async(Machine.current, thread_sensitive=True)()
-        with _perf_span("archivebox.MachineService.on_MachineEvent.prepare_config"):
-            old_config = dict(machine.config or {})
-            config = dict(old_config)
+        machine = await sync_to_async(Machine.current, thread_sensitive=True)()
+        old_config = dict(machine.config or {})
+        config = dict(old_config)
 
-        with _perf_span("archivebox.MachineService.on_MachineEvent.merge_config"):
-            if event.config is not None:
-                binary_only = _strip_to_binary_keys(event.config)
-                config.update(binary_only)
-            elif event.method == "update":
-                key = event.key.replace("config/", "", 1).strip()
-                if key and _is_binary_event_key(key):
-                    config[key] = event.value
-            elif event.method == "unset":
-                key = event.key.replace("config/", "", 1).strip()
-                if key and _is_binary_event_key(key):
-                    config.pop(key, None)
-            else:
-                return
+        if event.config is not None:
+            binary_only = _strip_to_binary_keys(event.config)
+            config.update(binary_only)
+        elif event.method == "update":
+            key = event.key.replace("config/", "", 1).strip()
+            if key and _is_binary_event_key(key):
+                config[key] = event.value
+        elif event.method == "unset":
+            key = event.key.replace("config/", "", 1).strip()
+            if key and _is_binary_event_key(key):
+                config.pop(key, None)
+        else:
+            return
 
         if config == old_config:
             return
         machine.config = config
-        with _perf_span("archivebox.MachineService.on_MachineEvent.save"):
-            await machine.asave(update_fields=["config", "modified_at"])
+        await machine.asave(update_fields=["config", "modified_at"])

archivebox/services/process_service.py

@@ -1,12 +1,7 @@
 from __future__ import annotations
 
 import asyncio
-import os
-import sys
-import time
-from contextlib import contextmanager
 from datetime import datetime
-from functools import wraps
 from typing import ClassVar
 
 from asgiref.sync import sync_to_async
@@ -17,41 +12,6 @@ from abx_dl.events import CrawlCleanupEvent, CrawlCompletedEvent, ProcessComplet
 from abx_dl.services.base import BaseService
 
 
-def _perf_trace(label):
-    def decorator(func):
-        if asyncio.iscoroutinefunction(func):
-
-            @wraps(func)
-            async def async_wrapper(*args, **kwargs):
-                if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
-                    return await func(*args, **kwargs)
-                started_at = time.perf_counter()
-                try:
-                    return await func(*args, **kwargs)
-                finally:
-                    elapsed_ms = (time.perf_counter() - started_at) * 1000
-                    print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
-
-            return async_wrapper
-
-        return func
-
-    return decorator
-
-
-@contextmanager
-def _perf_span(label: str):
-    if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
-        yield
-        return
-    started_at = time.perf_counter()
-    try:
-        yield
-    finally:
-        elapsed_ms = (time.perf_counter() - started_at) * 1000
-        print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
-
-
 def parse_event_datetime(value: str | None):
     if not value:
         return None
@@ -95,85 +55,76 @@ class ProcessService(BaseService):
             self._iface = await sync_to_async(current_network_interface_with_machine, thread_sensitive=True)()
         return self._iface
 
-    @_perf_trace("archivebox.ProcessService.on_ProcessStartedEvent__save_to_db")
     async def on_ProcessStartedEvent__save_to_db(self, event: ProcessStartedEvent) -> None:
         from archivebox.machine.models import Process
 
-        with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.current_iface"):
-            iface = await self.current_iface()
-        with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.prepare_fields"):
-            process_type = event.process_type or Process.TypeChoices.HOOK
-            worker_type = event.worker_type or ""
-            started_at = parse_event_datetime(event.start_ts)
+        iface = await self.current_iface()
+        process_type = event.process_type or Process.TypeChoices.HOOK
+        worker_type = event.worker_type or ""
+        started_at = parse_event_datetime(event.start_ts)
         if started_at is None:
             raise ValueError("ProcessStartedEvent.start_ts is required")
-        with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.lookup"):
-            if event.pid:
-                process_query = Process.objects.filter(pid=event.pid, started_at=started_at)
-            else:
-                process_query = Process.objects.filter(
-                    process_type=process_type,
-                    worker_type=worker_type,
-                    pwd=event.output_dir,
-                    started_at=started_at,
-                )
-            process = await process_query.order_by("-modified_at").afirst()
+        if event.pid:
+            process_query = Process.objects.filter(pid=event.pid, started_at=started_at)
+        else:
+            process_query = Process.objects.filter(
+                process_type=process_type,
+                worker_type=worker_type,
+                pwd=event.output_dir,
+                started_at=started_at,
+            )
+        process = await process_query.order_by("-modified_at").afirst()
         if process is None:
-            with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.create"):
-                process = await Process.objects.acreate(
-                    machine=iface.machine,
-                    iface=iface,
-                    process_type=process_type,
-                    worker_type=worker_type,
-                    pwd=event.output_dir,
-                    cmd=[event.hook_path, *event.hook_args],
-                    env=event.env,
-                    timeout=event.timeout,
-                    pid=event.pid or None,
-                    url=event.url or None,
-                    started_at=started_at,
-                    status=Process.StatusChoices.RUNNING,
-                    retry_at=None,
-                )
+            process = await Process.objects.acreate(
+                machine=iface.machine,
+                iface=iface,
+                process_type=process_type,
+                worker_type=worker_type,
+                pwd=event.output_dir,
+                cmd=[event.hook_path, *event.hook_args],
+                env=event.env,
+                timeout=event.timeout,
+                pid=event.pid or None,
+                url=event.url or None,
+                started_at=started_at,
+                status=Process.StatusChoices.RUNNING,
+                retry_at=None,
+            )
         elif process.iface_id != iface.id or process.machine_id != iface.machine_id:
             process.iface = iface
             process.machine = iface.machine
-            with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.update_iface"):
-                await process.asave(update_fields=["iface", "machine", "modified_at"])
+            await process.asave(update_fields=["iface", "machine", "modified_at"])
 
-        with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.assign_fields"):
-            process.pwd = event.output_dir
-            process.cmd = [event.hook_path, *event.hook_args]
-            process.env = event.env
-            process.timeout = event.timeout
-            process.pid = event.pid or None
-            process.url = event.url or process.url
-            process.process_type = process_type or process.process_type
-            process.worker_type = worker_type or process.worker_type
-            process.started_at = started_at
-            process.status = process.StatusChoices.RUNNING
-            process.retry_at = None
-        with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.hydrate_binary"):
-            await sync_to_async(process.hydrate_binary_from_context, thread_sensitive=True)(
-                plugin_name=event.plugin_name,
-                hook_path=event.hook_path,
-            )
-        with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.update"):
-            await Process.objects.filter(id=process.id).aupdate(
-                pwd=process.pwd,
-                cmd=process.cmd,
-                env=process.env,
-                timeout=process.timeout,
-                pid=process.pid,
-                url=process.url,
-                process_type=process.process_type,
-                worker_type=process.worker_type,
-                started_at=process.started_at,
-                status=process.status,
-                retry_at=process.retry_at,
-                binary_id=process.binary_id,
-                modified_at=timezone.now(),
-            )
+        process.pwd = event.output_dir
+        process.cmd = [event.hook_path, *event.hook_args]
+        process.env = event.env
+        process.timeout = event.timeout
+        process.pid = event.pid or None
+        process.url = event.url or process.url
+        process.process_type = process_type or process.process_type
+        process.worker_type = worker_type or process.worker_type
+        process.started_at = started_at
+        process.status = process.StatusChoices.RUNNING
+        process.retry_at = None
+        await sync_to_async(process.hydrate_binary_from_context, thread_sensitive=True)(
+            plugin_name=event.plugin_name,
+            hook_path=event.hook_path,
+        )
+        await Process.objects.filter(id=process.id).aupdate(
+            pwd=process.pwd,
+            cmd=process.cmd,
+            env=process.env,
+            timeout=process.timeout,
+            pid=process.pid,
+            url=process.url,
+            process_type=process.process_type,
+            worker_type=process.worker_type,
+            started_at=process.started_at,
+            status=process.status,
+            retry_at=process.retry_at,
+            binary_id=process.binary_id,
+            modified_at=timezone.now(),
+        )
 
     async def _completed_worker_loop(self) -> None:
         while True:
@@ -202,70 +153,63 @@ class ProcessService(BaseService):
     async def on_CrawlCompletedEvent__flush_completed(self, event: CrawlCompletedEvent) -> None:
         await self.flush_completed()
 
-    @_perf_trace("archivebox.ProcessService._save_completed_process_to_db")
     async def _save_completed_process_to_db(self, event: ProcessCompletedEvent) -> None:
         from archivebox.machine.models import Process
 
-        with _perf_span("archivebox.ProcessService._save_completed.current_iface"):
-            iface = await self.current_iface()
-        with _perf_span("archivebox.ProcessService._save_completed.prepare_fields"):
-            process_type = event.process_type or Process.TypeChoices.HOOK
-            worker_type = event.worker_type or ""
-            started_at = parse_event_datetime(event.start_ts)
+        iface = await self.current_iface()
+        process_type = event.process_type or Process.TypeChoices.HOOK
+        worker_type = event.worker_type or ""
+        started_at = parse_event_datetime(event.start_ts)
         if started_at is None:
             raise ValueError("ProcessCompletedEvent.start_ts is required")
-        with _perf_span("archivebox.ProcessService._save_completed.lookup"):
-            if event.pid:
-                process_query = Process.objects.filter(pid=event.pid, started_at=started_at)
-            else:
-                process_query = Process.objects.filter(
-                    process_type=process_type,
-                    worker_type=worker_type,
-                    pwd=event.output_dir,
-                    started_at=started_at,
-                )
-            process = await process_query.order_by("-modified_at").afirst()
+        if event.pid:
+            process_query = Process.objects.filter(pid=event.pid, started_at=started_at)
+        else:
+            process_query = Process.objects.filter(
+                process_type=process_type,
+                worker_type=worker_type,
+                pwd=event.output_dir,
+                started_at=started_at,
+            )
+        process = await process_query.order_by("-modified_at").afirst()
         if process is None:
-            with _perf_span("archivebox.ProcessService._save_completed.create_missing"):
-                await Process.objects.acreate(
-                    machine=iface.machine,
-                    iface=iface,
-                    process_type=process_type,
-                    worker_type=worker_type,
-                    pwd=event.output_dir,
-                    cmd=[event.hook_path, *event.hook_args],
-                    env=event.env,
-                    timeout=event.timeout,
-                    pid=event.pid or None,
-                    url=event.url or None,
-                    started_at=started_at,
-                    status=Process.StatusChoices.RUNNING,
-                    retry_at=None,
-                )
-                process = await process_query.order_by("-modified_at").afirst()
+            await Process.objects.acreate(
+                machine=iface.machine,
+                iface=iface,
+                process_type=process_type,
+                worker_type=worker_type,
+                pwd=event.output_dir,
+                cmd=[event.hook_path, *event.hook_args],
+                env=event.env,
+                timeout=event.timeout,
+                pid=event.pid or None,
+                url=event.url or None,
+                started_at=started_at,
+                status=Process.StatusChoices.RUNNING,
+                retry_at=None,
+            )
+            process = await process_query.order_by("-modified_at").afirst()
             if process is None:
                 return
 
-        with _perf_span("archivebox.ProcessService._save_completed.prepare_updates"):
-            missing_cmd = not process.cmd
-            updates = {
-                "machine_id": iface.machine_id,
-                "iface_id": iface.id,
-                "pwd": event.output_dir,
-                "pid": event.pid or process.pid,
-                "url": event.url or process.url,
-                "process_type": process_type or process.process_type,
-                "worker_type": worker_type or process.worker_type,
-                "started_at": started_at,
-                "ended_at": parse_event_datetime(event.end_ts) or timezone.now(),
-                "stdout": event.stdout,
-                "stderr": event.stderr,
-                "exit_code": event.exit_code,
-                "status": Process.StatusChoices.EXITED,
-                "retry_at": None,
-                "modified_at": timezone.now(),
-            }
-            if missing_cmd:
-                updates["cmd"] = [event.hook_path, *event.hook_args]
-        with _perf_span("archivebox.ProcessService._save_completed.update"):
-            await Process.objects.filter(id=process.id).aupdate(**updates)
+        missing_cmd = not process.cmd
+        updates = {
+            "machine_id": iface.machine_id,
+            "iface_id": iface.id,
+            "pwd": event.output_dir,
+            "pid": event.pid or process.pid,
+            "url": event.url or process.url,
+            "process_type": process_type or process.process_type,
+            "worker_type": worker_type or process.worker_type,
+            "started_at": started_at,
+            "ended_at": parse_event_datetime(event.end_ts) or timezone.now(),
+            "stdout": event.stdout,
+            "stderr": event.stderr,
+            "exit_code": event.exit_code,
+            "status": Process.StatusChoices.EXITED,
+            "retry_at": None,
+            "modified_at": timezone.now(),
+        }
+        if missing_cmd:
+            updates["cmd"] = [event.hook_path, *event.hook_args]
+        await Process.objects.filter(id=process.id).aupdate(**updates)

archivebox/services/snapshot_service.py

@@ -1,11 +1,6 @@
 from __future__ import annotations
 
-import asyncio
 import sys
-import os
-import time
-from contextlib import contextmanager
-from functools import wraps
 
 from asgiref.sync import sync_to_async
 from django.utils import timezone
@@ -16,53 +11,6 @@ from abx_dl.limits import CrawlLimitState
 from abx_dl.services.base import BaseService
 
 
-def _perf_trace(label):
-    def decorator(func):
-        if asyncio.iscoroutinefunction(func):
-
-            @wraps(func)
-            async def async_wrapper(*args, **kwargs):
-                if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
-                    return await func(*args, **kwargs)
-                started_at = time.perf_counter()
-                try:
-                    return await func(*args, **kwargs)
-                finally:
-                    elapsed_ms = (time.perf_counter() - started_at) * 1000
-                    print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
-
-            return async_wrapper
-
-        @wraps(func)
-        def sync_wrapper(*args, **kwargs):
-            if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
-                return func(*args, **kwargs)
-            started_at = time.perf_counter()
-            try:
-                return func(*args, **kwargs)
-            finally:
-                elapsed_ms = (time.perf_counter() - started_at) * 1000
-                print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
-
-        return sync_wrapper
-
-    return decorator
-
-
-@contextmanager
-def _perf_span(label: str):
-    if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
-        yield
-        return
-    started_at = time.perf_counter()
-    try:
-        yield
-    finally:
-        elapsed_ms = (time.perf_counter() - started_at) * 1000
-        print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
-
-
-@_perf_trace("archivebox.SnapshotService.finalize_completed_snapshot")
 def finalize_completed_snapshot(
     snapshot_id: str,
     *,
@@ -71,40 +19,33 @@ def finalize_completed_snapshot(
 ) -> None:
     from archivebox.core.models import Snapshot
 
-    with _perf_span("archivebox.SnapshotService.finalize_completed_snapshot.snapshot_lookup"):
-        snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first()
+    snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first()
     if snapshot is None:
         return
 
     if snapshot.downloaded_at is None:
-        with _perf_span("archivebox.SnapshotService.finalize_completed_snapshot.mark_downloaded"):
-            snapshot.downloaded_at = timezone.now()
-            snapshot.save(update_fields=["downloaded_at", "modified_at"])
+        snapshot.downloaded_at = timezone.now()
+        snapshot.save(update_fields=["downloaded_at", "modified_at"])
 
-    with _perf_span("archivebox.SnapshotService.finalize_completed_snapshot.stop_reason"):
-        stop_reason = crawl_limit_stop_reason if crawl_limit_stop_reason is not None else _crawl_limit_stop_reason(snapshot.crawl)
+    stop_reason = crawl_limit_stop_reason if crawl_limit_stop_reason is not None else _crawl_limit_stop_reason(snapshot.crawl)
     if snapshot.crawl_id and stop_reason in ("crawl_max_size", "crawl_timeout"):
-        with _perf_span("archivebox.SnapshotService.finalize_completed_snapshot.seal_due_to_limits"):
-            Snapshot.objects.filter(
-                crawl_id=snapshot.crawl_id,
-                status=Snapshot.StatusChoices.QUEUED,
-            ).exclude(id=snapshot.id).update(
-                status=Snapshot.StatusChoices.SEALED,
-                retry_at=None,
-                modified_at=timezone.now(),
-            )
+        Snapshot.objects.filter(
+            crawl_id=snapshot.crawl_id,
+            status=Snapshot.StatusChoices.QUEUED,
+        ).exclude(id=snapshot.id).update(
+            status=Snapshot.StatusChoices.SEALED,
+            retry_at=None,
+            modified_at=timezone.now(),
+        )
 
     if snapshot.status == Snapshot.StatusChoices.QUEUED:
-        with _perf_span("archivebox.SnapshotService.finalize_completed_snapshot.tick_queued"):
-            snapshot.sm.tick()
-            snapshot.refresh_from_db()
+        snapshot.sm.tick()
+        snapshot.refresh_from_db()
     if snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.is_finished_processing():
-        with _perf_span("archivebox.SnapshotService.finalize_completed_snapshot.seal_finished"):
-            snapshot.sm.seal()
-            snapshot.refresh_from_db()
+        snapshot.sm.seal()
+        snapshot.refresh_from_db()
 
-    with _perf_span("archivebox.SnapshotService.finalize_completed_snapshot.write_index_jsonl"):
-        snapshot.write_index_jsonl(output_dir=output_dir)
+    snapshot.write_index_jsonl(output_dir=output_dir)
 
 
 def _crawl_limit_stop_reason(crawl) -> str:
@@ -129,43 +70,36 @@ class SnapshotService(BaseService):
         self.bus.on(SnapshotEvent, self.on_SnapshotEvent)
         self.bus.on(SnapshotCompletedEvent, self.on_SnapshotCompletedEvent)
 
-    @_perf_trace("archivebox.SnapshotService.on_SnapshotEvent")
     async def on_SnapshotEvent(self, event: SnapshotEvent) -> None:
         from archivebox.core.models import Snapshot
 
-        with _perf_span("archivebox.SnapshotService.on_SnapshotEvent.snapshot_lookup"):
-            snapshot = await Snapshot.objects.filter(id=event.snapshot_id, crawl_id=self.crawl_id).afirst()
+        snapshot = await Snapshot.objects.filter(id=event.snapshot_id, crawl_id=self.crawl_id).afirst()
 
         if snapshot is not None:
             if snapshot.is_paused:
                 return
             if snapshot.status == Snapshot.StatusChoices.QUEUED:
                 try:
-                    with _perf_span("archivebox.SnapshotService.on_SnapshotEvent.tick_queued"):
-                        await sync_to_async(snapshot.sm.tick, thread_sensitive=True)()
+                    await sync_to_async(snapshot.sm.tick, thread_sensitive=True)()
                 except ValidationError as err:
                     if "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs." not in str(err):
                         raise
-                    with _perf_span("archivebox.SnapshotService.on_SnapshotEvent.seal_invalid_internal_url"):
-                        await Snapshot.objects.filter(id=snapshot.id).aupdate(
-                            status=Snapshot.StatusChoices.SEALED,
-                            retry_at=None,
-                            modified_at=timezone.now(),
-                        )
+                    await Snapshot.objects.filter(id=snapshot.id).aupdate(
+                        status=Snapshot.StatusChoices.SEALED,
+                        retry_at=None,
+                        modified_at=timezone.now(),
+                    )
                     rprint(
                         f"[red][X] Refusing to archive ArchiveBox internal URL for security: {snapshot.url}[/red]",
                         file=sys.stderr,
                     )
                     return
-                with _perf_span("archivebox.SnapshotService.on_SnapshotEvent.refresh_after_tick"):
-                    await sync_to_async(snapshot.refresh_from_db, thread_sensitive=True)()
+                await sync_to_async(snapshot.refresh_from_db, thread_sensitive=True)()
             elif snapshot.status != Snapshot.StatusChoices.STARTED:
                 return
             if snapshot.status != Snapshot.StatusChoices.STARTED:
                 return
-            with _perf_span("archivebox.SnapshotService.on_SnapshotEvent.ensure_crawl_symlink"):
-                await sync_to_async(snapshot.ensure_crawl_symlink, thread_sensitive=True)()
+            await sync_to_async(snapshot.ensure_crawl_symlink, thread_sensitive=True)()
 
-    @_perf_trace("archivebox.SnapshotService.on_SnapshotCompletedEvent")
     async def on_SnapshotCompletedEvent(self, event: SnapshotCompletedEvent) -> None:
         await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(event.snapshot_id)

archivebox/templates/admin/base.html

@@ -2028,13 +2028,6 @@
                     const scope = document.querySelector(selector)
                     const addLink = scope?.querySelector('.object-tools a.addlink')
                     const actionButtons = scope?.querySelector('.actions-top .action-buttons')
-                    if (addLink && actionButtons && !actionButtons.querySelector('.admin-add-action')) {
-                        const addButton = addLink.cloneNode(true)
-                        addButton.classList.add('button', 'admin-add-action')
-                        addButton.classList.remove('addlink')
-                        addButton.textContent = '➕ Add'
-                        actionButtons.appendChild(addButton)
-                    }
                 })
                 const tagContainer = document.querySelector('.actions-tags')
                 if (tagContainer) {

archivebox/templates/admin/core/tag/change_list.html

@@ -517,7 +517,7 @@ document.addEventListener('DOMContentLoaded', function () {
             sort: typeof next.sort === 'string' ? next.sort : (sortSelect?.value || 'created_desc'),
             created_by: typeof next.created_by === 'string' ? next.created_by : (createdBySelect?.value || ''),
             year: typeof next.year === 'string' ? next.year : (yearSelect?.value || ''),
-            has_snapshots: typeof next.has_snapshots === 'string' ? next.has_snapshots : (shell.dataset.initialHasSnapshots || 'yes'),
+            has_snapshots: typeof next.has_snapshots === 'string' ? next.has_snapshots : (shell.dataset.initialHasSnapshots || 'all'),
         };
     }
 
@@ -554,7 +554,7 @@ document.addEventListener('DOMContentLoaded', function () {
             url.searchParams.delete('year');
         }
 
-        if (state.has_snapshots && state.has_snapshots !== 'yes') {
+        if (state.has_snapshots && state.has_snapshots !== 'all') {
             url.searchParams.set('has_snapshots', state.has_snapshots);
         } else {
             url.searchParams.delete('has_snapshots');
@@ -570,9 +570,7 @@ document.addEventListener('DOMContentLoaded', function () {
     }
 
     function renderCards(nextCards, state) {
-        cards = Array.isArray(nextCards) ? nextCards.filter(function (card) {
-            return Number(card && card.num_snapshots || 0) > 0;
-        }) : [];
+        cards = Array.isArray(nextCards) ? nextCards : [];
         setMeta(state || getCurrentState(), cards.length);
 
         if (!cards.length) {