release: archivebox 0.9.34rc15

Clean rebuild of docs/apidocs/archivebox/ via autodoc2:
- Removes 19 stale module pages whose source files no longer exist
  (cli_utils, host_utils, schedule_utils — renamed to *_util; actors,
  ideas, debugging, folders, legacy, progress_layout, tests_piping,
  config_tags, personas.runtime/views, orchestrator*, worker, tasks).
- Adds 29 new module pages for code that was added since the previous
  generation but not yet documented.
- Updates 100 existing pages to reflect API surface changes (e.g.
  ENABLED_PLUGINS Field removed, SNAPSHOTS_PER_PAGE clamp removed,
  default bumped to 50, etc.).

156 source modules <-> 156 apidoc files (zero drift). Build clean
under sphinx-build -W --keep-going.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.github/workflows/codeql.yml

@@ -19,6 +19,10 @@ on:
   schedule:
     - cron: '33 17 * * 6'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   analyze:
     name: Analyze (${{ matrix.language }})

.github/workflows/debian.yml

@@ -16,6 +16,10 @@ on:
 permissions:
   contents: write
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     runs-on: ubuntu-24.04

.github/workflows/docker.yml

@@ -5,17 +5,39 @@ on:
   workflow_call:
   push:
     branches:
-      - '**'
+      - dev
     tags:
       - 'v*'
   # pull_request:
 
 env:
-  DOCKER_IMAGE: archivebox-ci
+  DOCKERHUB_IMAGE: archivebox/archivebox
+  GHCR_IMAGE: ghcr.io/archivebox/archivebox
+
+permissions:
+  contents: read
+  packages: write
+
+concurrency:
+  group: docker-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  buildx:
-    runs-on: ubuntu-latest
+  build:
+    name: build ${{ matrix.platform }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-24.04
+            cache_scope: docker-amd64
+            artifact_name: digest-linux-amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            cache_scope: docker-arm64
+            artifact_name: digest-linux-arm64
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -23,16 +45,12 @@ jobs:
         #   submodules: true
         #   fetch-depth: 1
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
         with:
           version: latest
           install: true
-          platforms: linux/amd64,linux/arm64
 
       - name: Builder instance name
         run: echo ${{ steps.buildx.outputs.name }}
@@ -40,14 +58,6 @@ jobs:
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
-      - name: Cache Docker layers
-        uses: actions/cache@v4
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         if: github.event_name != 'pull_request'
@@ -62,68 +72,190 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Collect Full Release Docker tags
-        # https://github.com/docker/metadata-action
+      - name: Collect Docker labels
         id: docker_meta
-        uses: docker/metadata-action@v5
-        if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
-        with:
-          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
-          tags: |
-              # :stable
-              type=ref,event=branch
-              # :0.7.3
-              type=semver,pattern={{version}}
-              # :0.7
-              type=semver,pattern={{major}}.{{minor}}
-              # :sha-463ea54
-              type=sha
-              # :latest
-              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$(python3 - <<'PY'
+          from pathlib import Path
+          import re
 
-      - name: Collect Non-Release Docker tags
-        # https://github.com/docker/metadata-action
-        id: docker_meta_non_release
-        uses: docker/metadata-action@v5
-        if: github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call'
-        with:
-          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
-          tags: |
-              # :stable
-              type=ref,event=branch
-              # :sha-463ea54
-              type=sha
+          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
+          if not match:
+              raise SystemExit("Failed to read version from pyproject.toml")
+          print(match.group(1))
+          PY
+          )"
 
-      - name: Build and push
+          {
+            echo 'labels<<EOF'
+            echo "org.opencontainers.image.version=${VERSION}"
+            echo "org.opencontainers.image.revision=${GITHUB_SHA}"
+            echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
+            echo 'EOF'
+            echo "version=${VERSION}"
+          } >> "$GITHUB_OUTPUT"
+
+          echo "[+] Building ${{ matrix.platform }} for ${VERSION}"
+
+      - name: Build and push digest
         id: docker_build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./
           file: ./Dockerfile
           builder: ${{ steps.buildx.outputs.name }}
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.tags || steps.docker_meta_non_release.outputs.tags }}
-          labels: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.labels || steps.docker_meta_non_release.outputs.labels }}
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new
-          platforms: linux/amd64,linux/arm64
+          tags: |
+            ${{ env.DOCKERHUB_IMAGE }}
+            ${{ env.GHCR_IMAGE }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+          cache-from: type=gha,scope=${{ matrix.cache_scope }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
+          platforms: ${{ matrix.platform }}
+          outputs: type=image,push-by-digest=true,name-canonical=true,push=true
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
 
+      - name: Export digest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mkdir -p /tmp/digests
+          digest="${{ steps.docker_build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.artifact_name }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  publish:
+    name: publish multiarch tags
+    runs-on: ubuntu-24.04
+    needs:
+      - build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: latest
+          install: true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digest-*
+          merge-multiple: true
+
+      - name: Collect Docker tags
+        id: docker_meta
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$(python3 - <<'PY'
+          from pathlib import Path
+          import re
+
+          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
+          if not match:
+              raise SystemExit("Failed to read version from pyproject.toml")
+          print(match.group(1))
+          PY
+          )"
+          SHORT_SHA="${GITHUB_SHA::8}"
+
+          {
+            echo 'dockerhub_tags<<EOF'
+            echo "${DOCKERHUB_IMAGE}:dev"
+            echo "${DOCKERHUB_IMAGE}:${VERSION}"
+            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+            echo 'EOF'
+            echo 'ghcr_tags<<EOF'
+            echo "${GHCR_IMAGE}:dev"
+            echo "${GHCR_IMAGE}:${VERSION}"
+            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+            echo 'EOF'
+            echo "version=${VERSION}"
+            echo "short_sha=${SHORT_SHA}"
+          } >> "$GITHUB_OUTPUT"
+
+          echo "[+] Publishing Docker Hub tags:"
+          printf '%s\n' "${DOCKERHUB_IMAGE}:dev" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+          echo "[+] Publishing GHCR tags:"
+          printf '%s\n' "${GHCR_IMAGE}:dev" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+
+      - name: Create Docker Hub manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
+          [[ "${#DIGESTS[@]}" -gt 0 ]]
+
+          TAG_ARGS=()
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
+          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
+
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do
+            REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}")
+          done
+
+          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Create GHCR manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
+          [[ "${#DIGESTS[@]}" -gt 0 ]]
+
+          TAG_ARGS=()
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
+          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
+
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do
+            REFS+=("${GHCR_IMAGE}@sha256:${digest}")
+          done
+
+          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Inspect published images
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
+          docker buildx imagetools inspect "${GHCR_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
+
       - name: Update README
         uses: peter-evans/dockerhub-description@v4
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: archivebox/archivebox
-
-      # This ugly bit is necessary if you don't want your cache to grow forever
-      # until it hits GitHub's limit of 5GB.
-      # Temp fix
-      # https://github.com/docker/build-push-action/issues/252
-      # https://github.com/moby/buildkit/issues/1896
-      - name: Move cache
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/homebrew.yml

@@ -16,6 +16,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build-and-test:
     strategy:

.github/workflows/lint.yml

@@ -4,6 +4,10 @@ on:
   workflow_dispatch:
   push:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   UV_NO_SOURCES: "1"
   PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
@@ -39,7 +43,7 @@ jobs:
 
       - name: Install dependencies with uv
         run: |
-          uv sync --all-extras --all-groups --no-sources --no-cache
+          uv sync --extra sonic --extra debug --all-groups --no-sources --no-cache
           uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
 
       - name: Run prek

.github/workflows/pip.yml

@@ -9,6 +9,10 @@ on:
     tags:
       - 'v*'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   PYTHON_VERSION: "3.13"
 
@@ -22,7 +26,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v6
         with:
           enable-cache: false
 

.github/workflows/release-runner.yml

@@ -10,6 +10,10 @@ permissions:
   contents: write
   id-token: write
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   release-state:
     runs-on: ubuntu-latest

.github/workflows/release.yml

@@ -18,6 +18,10 @@ permissions:
   packages: write
   id-token: write
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   pip:
     name: Publish to PyPI

.github/workflows/test-parallel.yml

@@ -1,11 +1,16 @@
 name: Parallel Tests
 
 on:
+  workflow_dispatch:
   pull_request:
     branches: [dev, main, master]
   push:
     branches: [dev]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   PYTHONIOENCODING: utf-8
   PYTHONLEGACYWINDOWSSTDIO: utf-8
@@ -15,7 +20,7 @@ env:
 jobs:
   discover-tests:
     name: Discover test files
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     outputs:
       test-files: ${{ steps.set-matrix.outputs.test-files }}
       plugin-tests: ${{ steps.set-plugin-matrix.outputs.plugin-tests }}
@@ -85,7 +90,7 @@ jobs:
 
   run-tests:
     name: ${{ matrix.test.name }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: discover-tests
     env:
       PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
@@ -116,13 +121,13 @@ jobs:
         run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
 
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
           architecture: x64
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v6
         with:
           version: "latest"
 
@@ -132,7 +137,7 @@ jobs:
           node-version: 22
 
       - name: Cache uv
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/uv
           key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
@@ -156,7 +161,7 @@ jobs:
 
   plugin-tests:
     name: ${{ matrix.plugin.name }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: discover-tests
     env:
       PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
@@ -186,13 +191,13 @@ jobs:
         run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
 
       - name: Set up Python 3.13
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.13"
           architecture: x64
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v6
         with:
           version: "latest"
 
@@ -202,7 +207,7 @@ jobs:
           node-version: 22
 
       - name: Cache uv
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/uv
           key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }}

.github/workflows/test.yml

@@ -1,6 +1,10 @@
 name: Run tests
 on: [push]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   DOCKER_IMAGE: archivebox-ci
   PYTHONIOENCODING: utf-8

.gitignore

@@ -63,3 +63,6 @@ ArchiveBox.conf
 # vim
 *.sw?
 .vscode
+
+# Local Claude Code task locks (not project state)
+.claude/scheduled_tasks.lock

.pre-commit-config.yaml

@@ -29,12 +29,12 @@ repos:
     hooks:
       - id: ruff-format
         name: ruff-format
-        entry: uv run --active --no-sources ruff format
+        entry: uv run --active --no-sync --no-sources ruff format
         language: system
         types_or: [python, pyi]
       - id: ruff-check
         name: ruff-check
-        entry: uv run --active --no-sources ruff check --fix
+        entry: uv run --active --no-sync --no-sources ruff check --fix
         language: system
         types_or: [python, pyi]
 

Dockerfile

@@ -86,8 +86,8 @@ ENV ARCHIVEBOX_USER="archivebox" \
 ENV CODE_DIR=/app \
     DATA_DIR=/data \
     LIB_DIR=/opt/archivebox/lib \
-    LIB_BIN_DIR=/opt/archivebox/lib/bin \
     ABXPKG_LIB_DIR=/opt/archivebox/lib \
+    SONIC_BINARY=/opt/archivebox/lib/env/bin/sonic \
     PLAYWRIGHT_BROWSERS_PATH=/browsers
 
 # Bash SHELL config
@@ -140,7 +140,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
         # 1. packaging dependencies
         apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
         # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep dnsutils git python3.12-venv \
+        zlib1g-dev dumb-init gosu cron unzip grep dnsutils git ripgrep python3.12-venv default-jre-headless \
         # 3. frivolous CLI helpers to make debugging failed archiving easier
         tree nano iputils-ping \
         # nano iputils-ping dnsutils htop procps jq yq
@@ -310,15 +310,37 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \
     --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
     echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
+    && export PERSONAS_DIR="$LIB_DIR/personas" \
+    && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
+    && mkdir -p "$LIB_DIR" \
     && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends build-essential tesseract-ocr tesseract-ocr-eng \
+    && abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" gallery-dl \
     && if [ "$TARGETARCH" = "arm64" ]; then \
-        abxpkg install --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
+        abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \
+        abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
     fi \
-    && PUID=0 PGID=0 abx-dl plugins --install \
-    && find "$LIB_DIR" "$DATA_DIR"/personas -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$LIB_DIR" "$DATA_DIR"/personas -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && TIMEOUT=600 PUID=0 PGID=0 abx-dl plugins --install \
+    && abxpkg install --no-cache --binproviders=chromewebstore --overrides='{"chromewebstore":{"install_args":["fpeoodllldobpkbkabpblcfaogecpndd","--name=archivewebpage"]}}' archivewebpage \
+    && test -f "$LIB_DIR/chromewebstore/extensions/fpeoodllldobpkbkabpblcfaogecpndd__archivewebpage/manifest.json" \
+    && mkdir -p "$LIB_DIR/env/bin" \
+    && ln -sf "$(command -v node)" "$LIB_DIR/env/bin/node" \
+    && ln -sf "$(command -v npm)" "$LIB_DIR/env/bin/npm" \
+    && ln -sf "$(command -v java)" "$LIB_DIR/env/bin/java" \
+    && ln -sf "$(command -v rg)" "$LIB_DIR/env/bin/rg" \
+    && ln -sf "$(command -v sonic)" "$LIB_DIR/env/bin/sonic" \
+    && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \
+    && mkdir -p /opt/archivebox/lib-layer \
+    && cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \
+    && apt-get purge -y build-essential \
+    && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/* \
-    && (chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR"/personas 2>/dev/null || true) \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer
+
+RUN rm -rf "$LIB_DIR" \
+    && mv /opt/archivebox/lib-layer "$LIB_DIR" \
     && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"
 
 # Install ArchiveBox Python package from the checked-out source.
@@ -362,7 +384,8 @@ RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary
 RUN chmod +x "$CODE_DIR"/bin/*.sh \
     && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
     && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
-    && gosu "$ARCHIVEBOX_USER" archivebox install 2>&1 | tee -a /VERSION.txt \
+    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install 2>&1 | tee -a /VERSION.txt \
+    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage gallerydl liteparse mercury papersdl readability search_backend_sonic opendataloader search_backend_ripgrep 2>&1 | tee -a /VERSION.txt \
     && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
     && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
     && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \

Dockerfile.multistage

@@ -0,0 +1,227 @@
+# syntax=docker/dockerfile:1.7
+
+# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
+# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
+# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
+# Build abx-dl first, then point this file at it:
+#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
+#       --build-context abxbus=../abxbus \
+#       --build-context abxpkg=../abxpkg \
+#       --build-context abx-plugins=../abx-plugins \
+#       -t archivebox/abx-dl:dev
+#   docker buildx build . -f Dockerfile.multistage \
+#       --build-context abx-dl=docker-image://archivebox/abx-dl:dev \
+#       --build-arg ABX_DL_IMAGE=abx-dl \
+#       -t archivebox:multistage
+
+ARG TARGETPLATFORM=linux/amd64
+ARG TARGETOS=linux
+ARG TARGETARCH=amd64
+ARG TARGETVARIANT=
+ARG ABX_DL_IMAGE=abx-dl
+
+FROM ${ABX_DL_IMAGE} AS abx-dl
+FROM archivebox/sonic:1.4.9 AS sonic
+FROM ubuntu:24.04 AS archivebox-runtime-base
+
+ARG TARGETPLATFORM=linux/amd64
+ARG TARGETOS=linux
+ARG TARGETARCH=amd64
+ARG TARGETVARIANT
+
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    npm_config_loglevel=error
+
+ENV PYTHON_VERSION=3.13 \
+    NODE_VERSION=22.22.3
+
+ENV ARCHIVEBOX_USER=archivebox \
+    DEFAULT_PUID=911 \
+    DEFAULT_PGID=911 \
+    IN_DOCKER=True
+
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    LIB_DIR=/opt/archivebox/lib \
+    ABXPKG_LIB_DIR=/opt/archivebox/lib \
+    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
+    PERSONAS_DIR=/data/personas \
+    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
+    CHROME_HEADLESS=true \
+    CHROME_SANDBOX=false \
+    CHROME_ISOLATION=crawl \
+    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
+
+ENV TMP_DIR=/tmp/archivebox \
+    PIP_VENV_PYTHON=/venv/bin/python3 \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no
+
+ENV UV_COMPILE_BYTECODE=0 \
+    UV_PYTHON_PREFERENCE=managed \
+    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv \
+    VIRTUAL_ENV=/venv \
+    PATH="/venv/bin:/opt/node/bin:$PATH"
+
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
+WORKDIR "$CODE_DIR"
+
+RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
+    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
+    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
+    && rm -f /etc/apt/apt.conf.d/docker-clean
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing ArchiveBox base runtime dependencies for $TARGETPLATFORM..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y \
+        apt-transport-https apt-utils ca-certificates curl wget gnupg2 \
+        dumb-init gosu unzip git grep ripgrep dnsutils iputils-ping procps tree nano \
+        cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \
+        libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \
+        libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \
+        libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 \
+        libxext6 libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1 \
+        fonts-liberation fonts-noto-color-emoji xdg-utils \
+        ffmpeg imagemagick tesseract-ocr tesseract-ocr-eng openjdk-21-jre-headless \
+    && rm -rf /var/lib/apt/lists/*
+
+# Runtime-owned layers copied from the abx-dl image.
+COPY --from=abx-dl /bin/uv /bin/uv
+COPY --from=abx-dl /opt/uv/python /opt/uv/python
+COPY --from=abx-dl /opt/node /opt/node
+COPY --from=abx-dl /venv /venv
+COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt
+
+RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && uname -a \
+    && sed -n '1,7p' /etc/os-release \
+    && which python3 && python3 --version \
+    && which node && node --version \
+    && which uv && uv self version \
+    ) | tee -a /VERSION.txt
+
+ENV PYTHONDONTWRITEBYTECODE=1
+
+FROM archivebox-runtime-base AS archivebox-builder
+
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
+    && uv sync \
+        --refresh \
+        --no-dev \
+        --inexact \
+        --all-extras \
+        --no-install-project \
+        --no-install-workspace \
+        --no-sources \
+    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
+    && apt-get autoremove -y \
+    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && uv pip install --no-deps "$CODE_DIR" \
+    && (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
+    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
+
+FROM archivebox-runtime-base
+
+LABEL name="archivebox" \
+    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
+    description="All-in-one self-hosted internet archiving solution" \
+    homepage="https://github.com/ArchiveBox/ArchiveBox" \
+    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
+    org.opencontainers.image.title="ArchiveBox" \
+    org.opencontainers.image.vendor="ArchiveBox" \
+    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
+    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
+    com.docker.image.source.entrypoint="Dockerfile.multistage"
+
+COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
+COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
+
+COPY --from=archivebox-builder /venv /venv
+COPY --from=archivebox-builder /app /app
+COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
+COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib
+
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
+    && groupadd --system "$ARCHIVEBOX_USER" \
+    && useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
+    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
+    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
+    && (which sonic && sonic --version) | tee -a /VERSION.txt \
+    && mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && openssl rand -hex 16 > /etc/machine-id \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
+    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
+
+WORKDIR "$DATA_DIR"
+RUN echo "[+] Initializing image collection..." \
+    && PUID=0 PGID=0 archivebox init \
+    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
+    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
+        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
+        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
+        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
+        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
+        2>/dev/null || true)
+
+RUN chmod +x "$CODE_DIR"/bin/*.sh \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
+    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install gallerydl liteparse mercury papersdl readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
+    && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
+    && "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
+    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
+    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
+    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
+    && ! command -v gcc \
+    && ! command -v g++ \
+    && ! command -v make \
+    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
+    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+
+RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
+    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
+    ) | tee -a /VERSION.txt
+
+WORKDIR "$DATA_DIR"
+VOLUME "$DATA_DIR"
+EXPOSE 8000
+
+HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
+    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
+
+ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
+CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]

README.md

@@ -106,7 +106,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
 <sub>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sub><br/>
-<sub>Set <code>LISTEN_HOST</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sub>
+<sub>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sub>
 </details>
 <br/>
 
@@ -588,7 +588,7 @@ docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 </code></pre>
 
 <sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
-<sup>Set <code>LISTEN_HOST</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sup>
+<sup>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sup>
 <br/><br/>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage">Usage: Web UI</a> wiki. ➡️</i>
 <br/><br/>

archivebox/__init__.py

@@ -16,8 +16,6 @@ import sys
 from pathlib import Path
 from typing import Protocol, cast
 
-from abx_plugins import get_plugins_dir
-
 
 class _ReconfigurableStream(Protocol):
     def reconfigure(self, *, line_buffering: bool) -> object: ...
@@ -59,38 +57,65 @@ check_not_root()
 check_not_inside_source_dir()
 check_io_encoding()
 
-# Install monkey patches for third-party libraries
-from .misc.monkey_patches import *  # noqa
-
-# Plugin directories
-BUILTIN_PLUGINS_DIR = Path(get_plugins_dir()).resolve()
-USER_PLUGINS_DIR = (
-    Path(
-        os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or os.environ.get("USER_PLUGINS_DIR") or os.environ.get("DATA_DIR", os.getcwd()),
-    )
-    / "custom_plugins"
-)
-
-# These are kept for backwards compatibility with existing code
-# that checks for plugins. The new hook system uses discover_hooks()
-ALL_PLUGINS = {
-    "builtin": BUILTIN_PLUGINS_DIR,
-    "user": USER_PLUGINS_DIR,
-}
-LOADED_PLUGINS = ALL_PLUGINS
-
-# Setup basic config, constants, paths, and version
-from .config.constants import CONSTANTS  # noqa
-from .config.paths import PACKAGE_DIR, DATA_DIR  # noqa
 from .config.version import VERSION  # noqa
 
-# Set MACHINE_ID env var so hook scripts can use it
-os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)
 
 __version__ = VERSION
 __author__ = "ArchiveBox"
 __license__ = "MIT"
 
+
+def __getattr__(name: str):
+    if name == "CONSTANTS":
+        from .config.constants import CONSTANTS
+
+        os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)
+        return CONSTANTS
+    if name == "DATA_DIR":
+        from .config.paths import DATA_DIR
+
+        return DATA_DIR
+    if name == "VERSION":
+        return VERSION
+    if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
+        from abx_plugins import get_plugins_dir
+
+        builtin_plugins_dir = Path(get_plugins_dir()).resolve()
+        user_plugins_dir = (
+            Path(
+                os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR")
+                or os.environ.get("USER_PLUGINS_DIR")
+                or os.environ.get("DATA_DIR", os.getcwd()),
+            )
+            / "custom_plugins"
+        )
+        plugins = {
+            "builtin": builtin_plugins_dir,
+            "user": user_plugins_dir,
+        }
+        values = {
+            "BUILTIN_PLUGINS_DIR": builtin_plugins_dir,
+            "USER_PLUGINS_DIR": user_plugins_dir,
+            "ALL_PLUGINS": plugins,
+            "LOADED_PLUGINS": plugins,
+        }
+        return values[name]
+    raise AttributeError(name)
+
+
+__all__ = (
+    "ASCII_LOGO",
+    "ASCII_ICON",
+    "PACKAGE_DIR",
+    "DATA_DIR",
+    "CONSTANTS",
+    "VERSION",
+    "BUILTIN_PLUGINS_DIR",
+    "USER_PLUGINS_DIR",
+    "ALL_PLUGINS",
+    "LOADED_PLUGINS",
+)
+
 ASCII_ICON = """
 ██████████████████████████████████████████████████████████████████████████████████████████████████ 
 ██████████████████████████████████████████████████████████████████████████████████████████████████ 

archivebox/api/auth.py

@@ -7,7 +7,7 @@ from django.http import HttpRequest
 from django.contrib.auth import authenticate
 from django.contrib.auth.models import User
 
-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth
+from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader
 from ninja.errors import HttpError
 
 
@@ -106,33 +106,6 @@ class QueryParamTokenAuth(APIKeyQuery):
         return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
 
 
-class UsernameAndPasswordAuth(HttpBasicAuth):
-    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
-
-    def authenticate(self, request: HttpRequest, username: str, password: str) -> User | None:
-        return _require_superuser(
-            auth_using_password(username=username, password=password, request=request),
-            request,
-            self.__class__.__name__,
-        )
-
-
-class DjangoSessionAuth:
-    """Allow authenticating with existing Django session cookies (same-origin only)."""
-
-    def __call__(self, request: HttpRequest) -> User | None:
-        return self.authenticate(request)
-
-    def authenticate(self, request: HttpRequest, **kwargs) -> User | None:
-        user = getattr(request, "user", None)
-        if isinstance(user, User) and user.is_authenticated:
-            setattr(request, "_api_auth_method", self.__class__.__name__)
-            if not user.is_superuser:
-                raise HttpError(403, "Valid session but User does not have permission (make sure user.is_superuser=True)")
-            return user
-        return None
-
-
 ### Enabled Auth Methods
 
 API_AUTH_METHODS = [

archivebox/api/models.py

@@ -1,7 +1,7 @@
 __package__ = "archivebox.api"
 
 import secrets
-from archivebox.uuid_compat import uuid7
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 
 from django.conf import settings
 from django.db import models
@@ -17,7 +17,7 @@ def generate_secret_token() -> str:
 
 
 class APIToken(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
@@ -41,7 +41,7 @@ class APIToken(models.Model):
 
 
 class OutboundWebhook(WebhookBase):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)

archivebox/api/urls.py

@@ -7,8 +7,7 @@ from django.shortcuts import redirect
 from django.urls import path
 from django.views.generic.base import RedirectView
 
-from archivebox.core.host_utils import build_web_url
-
+from archivebox.core.routes_util import build_web_url
 from .v1_api import urls as v1_api_urls
 
 

archivebox/api/v1_cli.py

@@ -8,10 +8,8 @@ from enum import Enum
 from django.http import HttpRequest
 
 from ninja import Router, Schema
-from pydantic import Field
 
 from archivebox.misc.util import ansi_to_html
-from archivebox.config.common import get_config
 
 
 # from .auth import API_AUTH_METHODS
@@ -61,9 +59,14 @@ class AddCommandSchema(Schema):
     snapshot_ids: list[str] | None = None
     tag: str = ""
     depth: int = 0
+    max_urls: int = 0
+    crawl_max_size: int = 0
+    crawl_timeout: int = 0
+    snapshot_max_size: int = 0
     parser: str = "auto"
     plugins: str = ""
-    update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
+    only_new: bool | None = None
+    update: bool = False
     overwrite: bool = False
     index_only: bool = False
 
@@ -88,8 +91,9 @@ class ScheduleCommandSchema(Schema):
     every: str | None = None
     tag: str = ""
     depth: int = 0
+    only_new: bool | None = None
+    update: bool = False
     overwrite: bool = False
-    update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
     clear: bool = False
 
 
@@ -107,7 +111,6 @@ class ListCommandSchema(Schema):
 
 
 class RemoveCommandSchema(Schema):
-    delete: bool = True
     after: float | None = 0
     before: float | None = 999999999999999
     filter_type: str = FilterTypeChoices.exact
@@ -118,18 +121,26 @@ class RemoveCommandSchema(Schema):
 def cli_add(request: HttpRequest, args: AddCommandSchema):
     from archivebox.cli.archivebox_add import add
 
+    config_overrides: dict[str, object] = {}
+    if args.only_new is not None:
+        config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
     crawl, snapshots = add(
         urls=args.urls,
         snapshot_ids=args.snapshot_ids,
         tag=args.tag,
         depth=args.depth,
-        update=args.update,
+        max_urls=args.max_urls,
+        crawl_max_size=args.crawl_max_size,
+        crawl_timeout=args.crawl_timeout,
+        snapshot_max_size=args.snapshot_max_size,
         index_only=args.index_only,
-        overwrite=args.overwrite,
         plugins=args.plugins,
         parser=args.parser,
         bg=True,  # Always run in background for API calls
         created_by_id=request.user.pk,
+        config=config_overrides or None,
     )
 
     snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
@@ -164,6 +175,7 @@ def cli_update(request: HttpRequest, args: UpdateCommandSchema):
         resume=args.resume,
         batch_size=args.batch_size,
         continuous=args.continuous,
+        stop_daemon_stack=False,
     )
     stdout = getattr(request, "stdout", None)
     stderr = getattr(request, "stderr", None)
@@ -180,6 +192,11 @@ def cli_update(request: HttpRequest, args: UpdateCommandSchema):
 def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
     from archivebox.cli.archivebox_schedule import schedule
 
+    config_overrides: dict[str, object] = {}
+    if args.only_new is not None:
+        config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
     result = schedule(
         import_path=args.import_path,
         add=args.add,
@@ -191,8 +208,7 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
         every=args.every,
         tag=args.tag,
         depth=args.depth,
-        overwrite=args.overwrite,
-        update=args.update,
+        config=config_overrides or None,
     )
 
     stdout = getattr(request, "stdout", None)
@@ -262,7 +278,6 @@ def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
 
     remove(
         yes=True,  # no way to interactively ask for confirmation via API, so we force yes
-        delete=args.delete,
         snapshots=snapshots_to_remove,
         before=args.before,
         after=args.after,

archivebox/api/v1_core.py

@@ -11,8 +11,7 @@ from typing import Union, Any, Annotated
 from datetime import datetime, time
 
 from django.db import transaction
-from django.db.models import Model, Q, Sum
-from django.db.models.functions import Coalesce
+from django.db.models import Model, Q
 from django.http import HttpRequest, HttpResponse
 from django.http.multipartparser import MultiPartParser, MultiPartParserError
 from django.core.exceptions import ValidationError
@@ -29,11 +28,12 @@ from ninja.pagination import paginate, PaginationBase
 from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot, ArchiveResult, Tag
+from archivebox.core.permissions import public_snapshots_queryset
 from archivebox.api.auth import auth_using_token
 from archivebox.config.common import get_config
-from archivebox.core.host_utils import build_web_url
-from archivebox.misc.util import validate_url_length
-from archivebox.core.tag_utils import (
+from archivebox.core.routes_util import build_web_url
+from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
+from archivebox.core.tag_util import (
     add_snapshot_counts,
     build_tag_cards,
     delete_tag as delete_tag_record,
@@ -50,6 +50,8 @@ from archivebox.core.tag_utils import (
 )
 from archivebox.crawls.models import Crawl
 from archivebox.api.v1_crawls import CrawlSchema
+from archivebox.search.config import get_search_mode, get_search_mode_backend
+from archivebox.search.query import apply_snapshot_search
 
 
 router = Router(tags=["Core Models"])
@@ -359,23 +361,26 @@ def _get_snapshot_by_ref(snapshot_id: str):
         return queryset.get(_uuid_ref_query("id", snapshot_id))
 
 
-def _touch_archiveresult_snapshot(snapshot: Snapshot) -> None:
-    now = timezone.now()
-    snapshot_updates = ["modified_at"]
-    if snapshot.downloaded_at is None:
-        snapshot.downloaded_at = now
-        snapshot_updates.append("downloaded_at")
-    if snapshot.status != Snapshot.StatusChoices.SEALED:
-        snapshot.status = Snapshot.StatusChoices.SEALED
-        snapshot.retry_at = None
-        snapshot_updates.extend(["status", "retry_at"])
-    snapshot.save(update_fields=snapshot_updates)
+def _queue_archiveresult_snapshot_maintenance(snapshot: Snapshot) -> None:
+    """
+    Mark an uploaded ArchiveResult's Snapshot as dirty without finalizing it.
 
-    try:
-        snapshot.ensure_crawl_symlink(snapshot_dir=snapshot.output_dir)
-        snapshot.write_index_jsonl(output_dir=snapshot.output_dir)
-    except Exception:
-        pass
+    Upload API handlers are allowed to persist files and ArchiveResult rows, but
+    Snapshot save() side effects, sealing, symlink creation, and index/details
+    rewrites belong to the runner. retry_at is the scheduler signal the runner
+    already watches, so only bump rows that are final or otherwise invisible.
+    """
+    # ArchiveResult.save() updates parent snapshot health/mtime before this
+    # helper runs. Re-read the scheduler columns so the short CAS update below
+    # does not lose to our own earlier ArchiveResult write.
+    snapshot = Snapshot.objects.only("id", "status", "retry_at", "downloaded_at", "modified_at").get(id=snapshot.id)
+    now = timezone.now()
+    updates = {"modified_at": now}
+    if snapshot.downloaded_at is None:
+        updates["downloaded_at"] = now
+    if snapshot.status == Snapshot.StatusChoices.SEALED or snapshot.retry_at is None:
+        updates["retry_at"] = now
+    snapshot.safe_update(updates, refresh=False)
 
 
 def _merge_archiveresult_output_file_maps(results: list[ArchiveResult]) -> dict[str, dict[str, Any]]:
@@ -530,13 +535,11 @@ def create_archiveresult(
     parsed_output_json = _parse_archiveresult_output_json(output_json)
     hook = hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME
     matching_results = list(
-        ArchiveResult.objects.select_for_update()
-        .filter(
+        ArchiveResult.objects.filter(
             snapshot=snapshot,
             plugin=plugin_name,
             hook_name=hook,
-        )
-        .order_by("created_at", "id"),
+        ).order_by("created_at", "id"),
     )
     existing_result = matching_results[0] if matching_results else None
     existing_output_files = _merge_archiveresult_output_file_maps(matching_results)
@@ -547,29 +550,42 @@ def create_archiveresult(
         existing_output_files=existing_output_files,
         allow_empty=True,
     )
-    output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
-    output_file_paths = list(output_files.keys())
     now = timezone.now()
-    if (
-        existing_result
-        and normalized_status == ArchiveResult.StatusChoices.STARTED
-        and existing_result.status != ArchiveResult.StatusChoices.STARTED
-    ):
-        normalized_status = existing_result.status
 
     with transaction.atomic():
-        if existing_result:
+        matching_results = list(
+            ArchiveResult.objects.filter(
+                snapshot=snapshot,
+                plugin=plugin_name,
+                hook_name=hook,
+            ).order_by("created_at", "id"),
+        )
+        if matching_results:
+            existing_result = matching_results[0]
+            output_files = {
+                **_merge_archiveresult_output_file_maps(matching_results),
+                **output_files,
+            }
             duplicate_ids = [result.id for result in matching_results[1:]]
             if duplicate_ids:
                 ArchiveResult.objects.filter(id__in=duplicate_ids).delete()
             result = existing_result
         else:
+            existing_result = None
             result = ArchiveResult(
                 snapshot=snapshot,
                 plugin=plugin_name,
                 hook_name=hook,
             )
 
+        if (
+            existing_result
+            and normalized_status == ArchiveResult.StatusChoices.STARTED
+            and existing_result.status != ArchiveResult.StatusChoices.STARTED
+        ):
+            normalized_status = existing_result.status
+        output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
+        output_file_paths = list(output_files.keys())
         result.status = normalized_status
         result.output_str = output_str or (output_file_paths[0] if output_file_paths else "")
         result.output_json = parsed_output_json
@@ -581,7 +597,7 @@ def create_archiveresult(
         result.save()
 
     if result.status != ArchiveResult.StatusChoices.STARTED:
-        _touch_archiveresult_snapshot(snapshot)
+        _queue_archiveresult_snapshot_maintenance(snapshot)
     return result
 
 
@@ -622,13 +638,16 @@ def patch_archiveresult(
             normalized_status = latest_result.status
         result.status = normalized_status
         update_fields.append("status")
+    elif latest_result.status == ArchiveResult.StatusChoices.QUEUED and ArchiveResult.output_files_upload_complete(output_files):
+        result.status = ArchiveResult.StatusChoices.SUCCEEDED
+        update_fields.append("status")
     if output_json:
         result.output_json = _parse_archiveresult_output_json(output_json)
         update_fields.append("output_json")
 
     result.save(update_fields=update_fields)
     if result.status != ArchiveResult.StatusChoices.STARTED:
-        _touch_archiveresult_snapshot(result.snapshot)
+        _queue_archiveresult_snapshot_maintenance(result.snapshot)
 
     return result
 
@@ -671,7 +690,7 @@ class SnapshotSchema(Schema):
 
     @staticmethod
     def resolve_archive_size(obj):
-        return int(getattr(obj, "output_size_sum", obj.archive_size) or 0)
+        return int(obj.archive_size or 0)
 
     @staticmethod
     def resolve_output_size(obj):
@@ -689,6 +708,7 @@ class SnapshotSchema(Schema):
 
 
 class SnapshotUpdateSchema(Schema):
+    action: str | None = None
     status: str | None = None
     retry_at: datetime | None = None
     tags: list[str] | None = None
@@ -766,10 +786,10 @@ def _filter_snapshots_for_rss(
         )
         .filter(bookmarked_at__lte=before_dt)
     )
-
     crawl_id = crawl_id.strip()
     if crawl_id:
-        queryset = queryset.filter(crawl__id__icontains=crawl_id)
+        matching_crawl_pks = list(filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).values_list("pk", flat=True)[:100])
+        queryset = queryset.filter(crawl_id__in=matching_crawl_pks)
 
     created_by = created_by.strip()
     if created_by:
@@ -828,7 +848,7 @@ def _snapshots_rss_response(
 
 
 class SnapshotFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup(["id__icontains", "timestamp__startswith"])] = None
+    id: Annotated[str | None, FilterLookup(["id__istartswith", "id__iendswith", "timestamp__startswith"])] = None
     created_by_id: Annotated[str | None, FilterLookup("crawl__created_by_id")] = None
     created_by_username: Annotated[str | None, FilterLookup("crawl__created_by__username__icontains")] = None
     created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
@@ -837,10 +857,8 @@ class SnapshotFilterSchema(FilterSchema):
     modified_at: Annotated[datetime | None, FilterLookup("modified_at")] = None
     modified_at__gte: Annotated[datetime | None, FilterLookup("modified_at__gte")] = None
     modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
-    search: Annotated[
-        str | None,
-        FilterLookup(["url__icontains", "title__icontains", "tags__name__icontains", "id__icontains", "timestamp__startswith"]),
-    ] = None
+    search: str | None = None
+    search_mode: str | None = None
     url: Annotated[str | None, FilterLookup("url")] = None
     tag: Annotated[str | None, FilterLookup("tags__name")] = None
     title: Annotated[str | None, FilterLookup("title__icontains")] = None
@@ -848,14 +866,37 @@ class SnapshotFilterSchema(FilterSchema):
     bookmarked_at__gte: Annotated[datetime | None, FilterLookup("bookmarked_at__gte")] = None
     bookmarked_at__lt: Annotated[datetime | None, FilterLookup("bookmarked_at__lt")] = None
 
+    def filter_search(self, value: str | None) -> Q:
+        return Q()
+
+    def filter_search_mode(self, value: str | None) -> Q:
+        return Q()
+
 
 @router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
 @paginate(CustomPagination)
 def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
     """List all Snapshot entries matching these filters."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0))
-    return filters.filter(queryset).distinct()
+    queryset = filters.filter(Snapshot.objects.all()).distinct()
+    query = (filters.search or "").strip()
+    if not query:
+        return queryset
+
+    runtime_config = getattr(request, "archivebox_config", None)
+    search_mode = get_search_mode(filters.search_mode, config=runtime_config)
+    try:
+        return apply_snapshot_search(
+            queryset,
+            query,
+            search_mode=search_mode,
+            config=runtime_config,
+            include_id_matches=True,
+        )
+    except Exception:
+        if get_search_mode_backend(search_mode, config=runtime_config):
+            return queryset.none()
+        return apply_snapshot_search(queryset, query, search_mode="meta", config=runtime_config, include_id_matches=True)
 
 
 @router.get("/snapshots.rss", url_name="get_snapshots_rss")
@@ -880,7 +921,7 @@ def get_snapshots_rss(
 def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
     """Get a specific Snapshot by id."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0))
+    queryset = Snapshot.objects.all()
     try:
         return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
     except Snapshot.DoesNotExist:
@@ -902,7 +943,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
         raise HttpError(400, "depth must be between 0 and 4")
 
     if data.crawl_id:
-        crawl = Crawl.objects.get(id__icontains=data.crawl_id)
+        crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), data.crawl_id).get()
         crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
         tags = tags or crawl_tags
     else:
@@ -959,12 +1000,28 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
     try:
         snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
     except Snapshot.DoesNotExist:
-        snapshot = Snapshot.objects.get(Q(id__icontains=snapshot_id))
+        snapshot = filter_queryset_by_uuid_substring(Snapshot.objects.all(), snapshot_id).get()
 
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
+    action = payload.pop("action", None)
     tags = payload.pop("tags", None)
 
+    if action:
+        if action == "pause":
+            snapshot.pause()
+            setattr(request, "with_archiveresults", False)
+            return snapshot
+        if action in ("resume", "unpause"):
+            snapshot.resume()
+            setattr(request, "with_archiveresults", False)
+            return snapshot
+        if action == "cancel":
+            snapshot.cancel()
+            setattr(request, "with_archiveresults", False)
+            return snapshot
+        raise HttpError(400, f"Invalid action: {action}")
+
     if "status" in payload:
         if payload["status"] not in Snapshot.StatusChoices.values:
             raise HttpError(400, f"Invalid status: {payload['status']}")
@@ -980,7 +1037,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
     if tags is not None:
         snapshot.save_tags(normalize_tag_list(tags))
 
-    snapshot.save(update_fields=update_fields)
+    if payload.get("status") == Snapshot.StatusChoices.SEALED:
+        snapshot.cancel()
+    else:
+        snapshot.save(update_fields=update_fields)
     setattr(request, "with_archiveresults", False)
     return snapshot
 
@@ -1164,6 +1224,40 @@ class TagSnapshotResponseSchema(Schema):
     tag_name: str
 
 
+def _get_snapshot_for_tag_edit(snapshot_ref: str) -> Snapshot:
+    snapshot_ref = str(snapshot_ref or "").strip().lower()
+    if not snapshot_ref:
+        raise HttpError(400, "Snapshot id is required")
+
+    snapshot_qs = Snapshot.objects.only("id")
+    is_full_uuid = len(snapshot_ref.replace("-", "")) == 32 and all(char in "0123456789abcdef-" for char in snapshot_ref)
+    if is_full_uuid:
+        try:
+            return snapshot_qs.get(pk=snapshot_ref.replace("-", ""))
+        except (Snapshot.DoesNotExist, ValueError):
+            pass
+
+    if len(snapshot_ref) >= 14:
+        try:
+            return snapshot_qs.get(timestamp=snapshot_ref)
+        except Snapshot.DoesNotExist:
+            pass
+        except Snapshot.MultipleObjectsReturned:
+            snapshot = snapshot_qs.filter(timestamp=snapshot_ref).first()
+            if snapshot is not None:
+                return snapshot
+
+    try:
+        return snapshot_qs.get(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref))
+    except Snapshot.DoesNotExist:
+        raise HttpError(404, "Snapshot not found") from None
+    except Snapshot.MultipleObjectsReturned:
+        snapshot = snapshot_qs.filter(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref)).first()
+        if snapshot is None:
+            raise HttpError(404, "Snapshot not found")
+        return snapshot
+
+
 @router.get("/tags/search/", response=TagSearchResponseSchema, url_name="search_tags")
 def search_tags(
     request: HttpRequest,
@@ -1182,6 +1276,7 @@ def search_tags(
         "tags": build_tag_cards(
             query=q,
             request=request,
+            preview_limit=0,
             sort=normalized_sort,
             created_by=normalized_created_by,
             year=normalized_year,
@@ -1195,10 +1290,7 @@ def search_tags(
 
 
 def _public_tag_listing_enabled() -> bool:
-    config = get_config()
-    if config.PUBLIC_SNAPSHOTS_LIST is not None:
-        return config.PUBLIC_SNAPSHOTS_LIST
-    return config.PUBLIC_INDEX
+    return get_config().PUBLIC_INDEX
 
 
 def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
@@ -1223,8 +1315,13 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
     if not _request_has_tag_autocomplete_access(request):
         raise HttpError(401, "Authentication required")
 
-    tags = list(get_matching_tags(q, with_snapshot_counts=False)[: 50 if not q else 20])
-    add_snapshot_counts(tags)
+    public_only = not getattr(request.user, "is_authenticated", False) and not getattr(request, "_api_token", None)
+    queryset = get_matching_tags(q)
+    public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
+    if public_only:
+        queryset = queryset.filter(snapshot_set__id__in=public_snapshots.values("id")).distinct()
+    tags = list(queryset[: 50 if not q else 20])
+    add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)
 
     return {
         "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": getattr(tag, "num_snapshots", 0)} for tag in tags],
@@ -1308,19 +1405,7 @@ def tag_snapshots_export(request: HttpRequest, tag_id: int):
 @router.post("/tags/add-to-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_add_to_snapshot")
 def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
     """Add a tag to a snapshot. Creates the tag if it doesn't exist."""
-    # Get the snapshot
-    try:
-        snapshot = Snapshot.objects.get(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        )
-    except Snapshot.DoesNotExist:
-        raise HttpError(404, "Snapshot not found")
-    except Snapshot.MultipleObjectsReturned:
-        snapshot = Snapshot.objects.filter(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        ).first()
-        if snapshot is None:
-            raise HttpError(404, "Snapshot not found")
+    snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
 
     # Get or create the tag
     if data.tag_name:
@@ -1352,19 +1437,7 @@ def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
 @router.post("/tags/remove-from-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_remove_from_snapshot")
 def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
     """Remove a tag from a snapshot."""
-    # Get the snapshot
-    try:
-        snapshot = Snapshot.objects.get(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        )
-    except Snapshot.DoesNotExist:
-        raise HttpError(404, "Snapshot not found")
-    except Snapshot.MultipleObjectsReturned:
-        snapshot = Snapshot.objects.filter(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        ).first()
-        if snapshot is None:
-            raise HttpError(404, "Snapshot not found")
+    snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
 
     # Get the tag
     if data.tag_id:

archivebox/api/v1_crawls.py

@@ -1,8 +1,9 @@
 __package__ = "archivebox.api"
 
+from pathlib import Path
 from uuid import UUID
 from datetime import datetime
-from django.http import HttpRequest
+from django.http import FileResponse, HttpRequest
 from django.shortcuts import redirect
 from django.utils import timezone
 
@@ -13,10 +14,17 @@ from ninja import Router, Schema
 from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot
-from archivebox.crawls.models import Crawl
+from archivebox.core.permissions import (
+    PERMISSIONS_PUBLIC,
+    PERMISSIONS_UNLISTED,
+    is_admin_user,
+    normalize_permissions,
+)
 from archivebox.config.common import get_config
+from archivebox.crawls.models import Crawl
+from archivebox.misc.util import filter_queryset_by_uuid_substring
 
-from .auth import API_AUTH_METHODS
+from .auth import API_AUTH_METHODS, auth_using_token
 
 router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
 
@@ -33,22 +41,15 @@ class CrawlSchema(Schema):
 
     status: str
     retry_at: datetime | None
+    is_paused: bool
 
     urls: str
     max_depth: int
-    max_urls: int
-    crawl_max_size: int
-    snapshot_max_size: int
-    crawl_max_concurrent_snapshots: int
     tags_str: str
     config: dict
 
     # snapshots: List[SnapshotSchema]
 
-    @staticmethod
-    def resolve_crawl_max_concurrent_snapshots(obj):
-        return int((obj.config or {}).get("CRAWL_MAX_CONCURRENT_SNAPSHOTS") or get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS)
-
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
@@ -60,6 +61,14 @@ class CrawlSchema(Schema):
         username = getattr(user, "username", None)
         return username if isinstance(username, str) else str(user)
 
+    @staticmethod
+    def resolve_config(obj):
+        # Redact credential values so REST responses can never leak the raw
+        # token/secret/api-key that the operator stored in Crawl.config.
+        from archivebox.config.common import redact_sensitive_config
+
+        return redact_sensitive_config(obj.config)
+
     @staticmethod
     def resolve_snapshots(obj, context):
         if bool(getattr(context["request"], "with_snapshots", False)):
@@ -68,6 +77,7 @@ class CrawlSchema(Schema):
 
 
 class CrawlUpdateSchema(Schema):
+    action: str | None = None
     status: str | None = None
     retry_at: datetime | None = None
     tags: list[str] | None = None
@@ -77,10 +87,6 @@ class CrawlUpdateSchema(Schema):
 class CrawlCreateSchema(Schema):
     urls: list[str]
     max_depth: int = 0
-    max_urls: int = 0
-    crawl_max_size: int = 0
-    snapshot_max_size: int = 0
-    crawl_max_concurrent_snapshots: int | None = None
     tags: list[str] | None = None
     tags_str: str = ""
     label: str = ""
@@ -113,27 +119,14 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
         raise HttpError(400, "At least one URL is required")
     if data.max_depth not in (0, 1, 2, 3, 4):
         raise HttpError(400, "max_depth must be between 0 and 4")
-    if data.max_urls < 0:
-        raise HttpError(400, "max_urls must be >= 0")
-    if data.crawl_max_size < 0:
-        raise HttpError(400, "crawl_max_size must be >= 0")
-    if data.snapshot_max_size < 0:
-        raise HttpError(400, "snapshot_max_size must be >= 0")
-    crawl_max_concurrent_snapshots = data.crawl_max_concurrent_snapshots
-    if crawl_max_concurrent_snapshots is None:
-        crawl_max_concurrent_snapshots = get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS
-    if crawl_max_concurrent_snapshots < 1:
-        raise HttpError(400, "crawl_max_concurrent_snapshots must be >= 1")
 
     tags = normalize_tag_list(data.tags, data.tags_str)
     config = dict(data.config or {})
-    config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
+    request_user = request.user if request.user.is_authenticated else None
+    config.setdefault("PERMISSIONS", str(get_config(user=request_user).PERMISSIONS))
     crawl = Crawl.objects.create(
         urls="\n".join(urls),
         max_depth=data.max_depth,
-        max_urls=data.max_urls,
-        crawl_max_size=data.crawl_max_size,
-        snapshot_max_size=data.snapshot_max_size,
         tags_str=",".join(tags),
         label=data.label,
         notes=data.notes,
@@ -142,7 +135,6 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
         retry_at=timezone.now(),
         created_by=request.user if isinstance(request.user, User) else None,
     )
-    crawl.create_snapshots_from_urls()
     return crawl
 
 
@@ -151,7 +143,7 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn
     """Get a specific Crawl by id."""
     setattr(request, "with_snapshots", with_snapshots)
     setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
 
     if crawl and as_rss:
         query = request.GET.copy()
@@ -162,13 +154,87 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn
     return crawl
 
 
+def crawl_file(request: HttpRequest, crawl_id: str, path: str):
+    # Try to resolve the crawl first; if it doesn't exist, return 404.
+    try:
+        crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    except Crawl.DoesNotExist:
+        raise HttpError(404, "Crawl not found")
+
+    # Determine the effective viewer: session user takes precedence, otherwise
+    # fall back to an API token passed via ?api_key=, X-ArchiveBox-API-Key, or
+    # Authorization: Bearer ... (so that programmatic clients still work).
+    user = getattr(request, "user", None)
+    is_authenticated = bool(getattr(user, "is_authenticated", False) and getattr(user, "is_active", False))
+    if not is_authenticated:
+        token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
+        auth_header = request.headers.get("Authorization", "")
+        if not token and auth_header.lower().startswith("bearer "):
+            token = auth_header.split(None, 1)[1].strip()
+        token_user = auth_using_token(token=token, request=request) if token else None
+        if token_user and token_user.is_active:
+            user = token_user
+            is_authenticated = True
+            # Re-bind so is_admin_user() / ownership checks below see the token user.
+            setattr(request, "user", token_user)
+
+    # Gate access using the same model as SnapshotView/can_view_snapshot:
+    # admins always pass; owners can see their own crawls; otherwise the crawl
+    # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
+    if not is_admin_user(request):
+        permissions = normalize_permissions(crawl.permissions)
+        is_owner = bool(is_authenticated and getattr(crawl, "created_by_id", None) == getattr(user, "id", None))
+        if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
+            raise HttpError(404, "Crawl not found")
+
+    crawl_root = Path(crawl.output_dir).resolve()
+    file_path = (crawl_root / path).resolve()
+    if not file_path.is_file() or crawl_root not in file_path.parents:
+        raise HttpError(404, "Crawl file not found")
+
+    response = FileResponse(file_path.open("rb"))
+    response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
+    response["Pragma"] = "no-cache"
+    response["Expires"] = "0"
+    response["X-Content-Type-Options"] = "nosniff"
+    return response
+
+
+@router.get("/crawl/{crawl_id}/files/{filename}", auth=None, url_name="crawl_file_root")
+def crawl_file_root(request: HttpRequest, crawl_id: str, filename: str):
+    return crawl_file(request, crawl_id, filename)
+
+
+@router.get("/crawl/{crawl_id}/files/{folder}/{filename}", auth=None, url_name="crawl_file_nested_1")
+def crawl_file_nested_1(request: HttpRequest, crawl_id: str, folder: str, filename: str):
+    return crawl_file(request, crawl_id, f"{folder}/{filename}")
+
+
+@router.get("/crawl/{crawl_id}/files/{folder}/{subfolder}/{filename}", auth=None, url_name="crawl_file_nested_2")
+def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfolder: str, filename: str):
+    return crawl_file(request, crawl_id, f"{folder}/{subfolder}/{filename}")
+
+
 @router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
 def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
     """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
 
+    action = payload.pop("action", None)
+    if action:
+        if action == "pause":
+            crawl.pause()
+            return crawl
+        if action in ("resume", "unpause"):
+            crawl.resume()
+            return crawl
+        if action == "cancel":
+            crawl.cancel()
+            return crawl
+        raise HttpError(400, f"Invalid action: {action}")
+
     tags = payload.pop("tags", None)
     tags_str = payload.pop("tags_str", None)
     if tags is not None or tags_str is not None:
@@ -178,37 +244,23 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
     if "status" in payload:
         if payload["status"] not in Crawl.StatusChoices.values:
             raise HttpError(400, f"Invalid status: {payload['status']}")
+        if payload["status"] == Crawl.StatusChoices.SEALED:
+            crawl.cancel()
+            return crawl
         crawl.status = payload["status"]
-        if crawl.status == Crawl.StatusChoices.SEALED and "retry_at" not in payload:
-            crawl.retry_at = None
         update_fields.append("status")
 
     if "retry_at" in payload:
         crawl.retry_at = payload["retry_at"]
         update_fields.append("retry_at")
 
-    if payload.get("status") == Crawl.StatusChoices.SEALED:
-        cancelled_at = timezone.now()
-        crawl.retry_at = None
-        if "retry_at" not in update_fields:
-            update_fields.append("retry_at")
-        crawl.save(update_fields=update_fields)
-        Snapshot.objects.filter(
-            crawl=crawl,
-            status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
-        ).update(
-            status=Snapshot.StatusChoices.SEALED,
-            retry_at=None,
-            modified_at=cancelled_at,
-        )
-    else:
-        crawl.save(update_fields=update_fields)
+    crawl.save(update_fields=update_fields)
     return crawl
 
 
 @router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
 def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
     crawl_id_str = str(crawl.id)
     snapshot_count = crawl.snapshot_set.count()
     deleted_count, _ = crawl.delete()

archivebox/api/v1_personas.py

@@ -51,6 +51,14 @@ class PersonaSchema(Schema):
     def resolve_created_by_username(obj) -> str:
         return obj.created_by.username
 
+    @staticmethod
+    def resolve_config(obj):
+        # Redact credential values so REST responses don't leak the raw
+        # token/secret/api-key the operator stored in Persona.config.
+        from archivebox.config.common import redact_sensitive_config
+
+        return redact_sensitive_config(obj.config)
+
 
 class PersonaSyncResponseSchema(Schema):
     success: bool

archivebox/base_models/admin.py

@@ -3,6 +3,7 @@
 __package__ = "archivebox.base_models"
 
 import json
+import uuid
 from collections.abc import Mapping
 from typing import NotRequired, TypedDict
 
@@ -11,10 +12,41 @@ from django.contrib import admin
 from django.db import models
 from django.forms.renderers import BaseRenderer
 from django.http import HttpRequest, QueryDict
+from django.urls import path, register_converter
 from django.utils.safestring import SafeString, mark_safe
 from django_object_actions import DjangoObjectActions
 
 
+class HexUUIDConverter:
+    """URL path converter that canonicalizes UUIDs to their 32-char hex form.
+
+    Accepts both the hyphenated (``aaaaaaaa-bbbb-...``) and bare-hex
+    (``aaaaaaaabbbb...``) UUID strings on the way in (Django's UUIDField
+    parses either), but ``to_url`` always emits the bare-hex form. This is
+    what makes ``reverse("admin:app_model_change", args=[obj.pk])`` produce
+    ``/admin/app/model/06a1a8facb0d.../change/`` instead of the default
+    hyphenated rendering — admin links throughout the app reverse through
+    this converter once ``BaseModelAdmin.get_urls`` swaps in
+    ``<hexuuid:object_id>`` below.
+    """
+
+    regex = r"[0-9a-fA-F]{32}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
+
+    def to_python(self, value: str) -> str:
+        # Strip hyphens but stay as a string — Django admin treats the
+        # captured object_id as a string and calls ``model._meta.pk.to_python``
+        # itself, so we don't want to short-circuit that.
+        return value.replace("-", "")
+
+    def to_url(self, value) -> str:
+        if isinstance(value, uuid.UUID):
+            return value.hex
+        return str(value).replace("-", "")
+
+
+register_converter(HexUUIDConverter, "hexuuid")
+
+
 class ConfigOption(TypedDict):
     plugin: str
     type: str | list[str]
@@ -45,10 +77,10 @@ class KeyValueWidget(forms.Widget):
         """Get available config options from plugins."""
         try:
             from archivebox.config.common import ArchiveBoxConfig
-            from archivebox.hooks import discover_plugin_configs
+            from archivebox.plugins.discovery import discover_plugin_configs
 
             options: dict[str, ConfigOption] = {}
-            skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "CRAWL_OUTPUT_DIR", "SNAP_DIR"}
+            skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "SNAP_DIR"}
             for key, field in ArchiveBoxConfig.model_fields.items():
                 if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys:
                     continue
@@ -117,7 +149,7 @@ class KeyValueWidget(forms.Widget):
         config_meta_json = json.dumps(config_options)
 
         html = f'''
-        <div id="{widget_id}_container" class="key-value-editor" style="max-width: 700px;">
+        <div id="{widget_id}_container" class="key-value-editor" style="width: 100%; max-width: none;">
             <datalist id="{widget_id}_keys">
                 {datalist_options}
             </datalist>
@@ -227,9 +259,6 @@ class KeyValueWidget(forms.Widget):
                             return 'Example: ["value"]';
                         }}
                         if (types.includes('object')) {{
-                            if (key === 'SAVE_ALLOWLIST' || key === 'SAVE_DENYLIST') {{
-                                return 'Example: {{"^https://example\\\\.com": ["wget"]}}';
-                            }}
                             return 'Example: {{"key": "value"}}';
                         }}
                         return '';
@@ -238,8 +267,6 @@ class KeyValueWidget(forms.Widget):
                     function isRegexConfigKey_{widget_id}(key) {{
                         return key === 'URL_ALLOWLIST' ||
                             key === 'URL_DENYLIST' ||
-                            key === 'SAVE_ALLOWLIST' ||
-                            key === 'SAVE_DENYLIST' ||
                             key.endsWith('_PATTERN') ||
                             key.includes('REGEX');
                     }}
@@ -606,7 +633,7 @@ class KeyValueWidget(forms.Widget):
                         var newRow = document.createElement('div');
                         newRow.className = 'key-value-row';
                         newRow.style.cssText = 'margin-bottom: 6px;';
-                        newRow.innerHTML = '<div style="display: flex; gap: 8px; align-items: center;">' +
+                        newRow.innerHTML = '<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">' +
                             '<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
                             'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
                             '<input type="text" class="kv-value" placeholder="value" ' +
@@ -633,17 +660,74 @@ class KeyValueWidget(forms.Widget):
 
                     window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
 
+                    function focusConfigKeyFromHash_{widget_id}() {{
+                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
+                        // to (or creates) the matching row in this editor. Used by
+                        // the in-banner "pin via admin" link and the
+                        // ``→ Edit <KEY> in Machine.config`` shortcut on the live
+                        // config detail page.
+                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
+                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
+                            return;
+                        }}
+                        var container = document.getElementById('{widget_id}_rows');
+                        if (!container) {{
+                            return;
+                        }}
+                        var match = null;
+                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
+                            if (match) {{ return; }}
+                            var keyInput = row.querySelector('.kv-key');
+                            if (keyInput && keyInput.value.trim() === hash) {{
+                                match = row;
+                            }}
+                        }});
+                        if (!match) {{
+                            // No existing row for this key — prepopulate one with the
+                            // key filled in but value left blank so the operator just
+                            // types/pastes the value and hits save.
+                            window.addKeyValueRow_{widget_id}();
+                            var rows = container.querySelectorAll('.key-value-row');
+                            match = rows[rows.length - 1];
+                            var keyInput = match.querySelector('.kv-key');
+                            if (keyInput) {{
+                                keyInput.value = hash;
+                                keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
+                            }}
+                        }}
+                        if (!match) {{
+                            return;
+                        }}
+                        match.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
+                        var prevOutline = match.style.outline;
+                        match.style.outline = '2px solid #f59e0b';
+                        match.style.outlineOffset = '2px';
+                        match.style.transition = 'outline 1.2s ease-out';
+                        setTimeout(function() {{
+                            match.style.outline = prevOutline || 'none';
+                        }}, 1400);
+                        var valueInput = match.querySelector('.kv-value');
+                        if (valueInput) {{
+                            valueInput.focus();
+                            try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}}
+                        }}
+                    }}
+
                     // Initialize on load
                     document.addEventListener('DOMContentLoaded', function() {{
                         initializeRows_{widget_id}();
                         updateHiddenField_{widget_id}();
+                        focusConfigKeyFromHash_{widget_id}();
                     }});
                     // Also run immediately in case DOM is already ready
                     if (document.readyState !== 'loading') {{
                         initializeRows_{widget_id}();
                         updateHiddenField_{widget_id}();
+                        focusConfigKeyFromHash_{widget_id}();
                     }}
 
+                    window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id});
+
                     // Update on any input change
                     var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
 
@@ -670,12 +754,34 @@ class KeyValueWidget(forms.Widget):
         return mark_safe(html)
 
     def _render_row(self, widget_id: str, key: str, value: str) -> str:
+        from archivebox.config.common import is_sensitive_config_key
+
+        # Sensitive keys (``*TOKEN*``, ``*SECRET*``, ``*API_KEY*``, ``*APIKEY*``) are
+        # rendered write-only: the input is a password field with a placeholder
+        # showing the value is set, but the raw value is NEVER sent to the browser.
+        # When the user submits the form with the field left blank, the
+        # ``ConfigEditorMixin.save_model`` hook re-merges the previously-saved
+        # value so leaving it untouched is a no-op rather than a destructive clear.
+        is_sensitive = is_sensitive_config_key(key)
+        has_value = bool(value)
+        if is_sensitive:
+            input_type = "password"
+            rendered_value = ""
+            placeholder = (
+                "•••••• (saved — enter new value to replace, clear by deleting row)" if has_value else "value (will be saved write-only)"
+            )
+            extra_attrs = ' autocomplete="off" data-sensitive="1"' + (' data-had-value="1"' if has_value else "")
+        else:
+            input_type = "text"
+            rendered_value = self._escape(value)
+            placeholder = "value"
+            extra_attrs = ""
         return f'''
             <div class="key-value-row" style="margin-bottom: 6px;">
-                <div style="display: flex; gap: 8px; align-items: center;">
+                <div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">
                     <input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
                            style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <input type="text" class="kv-value" value="{self._escape(value)}" placeholder="value"
+                    <input type="{input_type}" class="kv-value" value="{rendered_value}" placeholder="{self._escape(placeholder)}"{extra_attrs}
                            style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
                     <datalist class="kv-value-options"></datalist>
                     <button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
@@ -710,7 +816,7 @@ class ConfigEditorMixin(admin.ModelAdmin):
 
     def formfield_for_dbfield(
         self,
-        db_field: models.Field[object, object],
+        db_field: models.Field,
         request: HttpRequest,
         **kwargs: object,
     ) -> forms.Field | None:
@@ -719,6 +825,37 @@ class ConfigEditorMixin(admin.ModelAdmin):
             kwargs["widget"] = KeyValueWidget()
         return super().formfield_for_dbfield(db_field, request, **kwargs)
 
+    def save_model(self, request: HttpRequest, obj, form, change):
+        """Preserve write-only redacted credentials on save.
+
+        The KeyValueWidget renders sensitive keys (``*TOKEN*``, ``*SECRET*``,
+        ``*API_KEY*``, ``*APIKEY*``) with an empty value + password input —
+        the real value never leaves the server. On submit, an empty value
+        for a sensitive key that was previously set means "leave untouched",
+        not "clear it." We honor that here by re-merging the stored value
+        before the row is written. Explicitly removing the row in the UI
+        still clears it (the key is gone from the submitted JSON, so there's
+        nothing to merge over).
+        """
+        from archivebox.config.common import is_sensitive_config_key
+
+        if change and obj.pk and getattr(obj, "config", None) is not None:
+            try:
+                stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
+            except Exception:
+                stored = {}
+            if isinstance(stored, dict):
+                new_config = dict(obj.config or {})
+                for key, new_value in list(new_config.items()):
+                    if not is_sensitive_config_key(key):
+                        continue
+                    if new_value not in (None, "") and new_value != "********":
+                        continue
+                    if key in stored:
+                        new_config[key] = stored[key]
+                obj.config = new_config
+        super().save_model(request, obj, form, change)
+
 
 class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
     list_display = ("id", "created_at", "created_by")
@@ -741,3 +878,30 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
         if "created_by" in form.base_fields:
             form.base_fields["created_by"].initial = request.user
         return form
+
+    def get_urls(self):
+        """Swap the per-object admin URLs from ``<path:object_id>`` to
+        ``<hexuuid:object_id>`` so canonical change/delete/history URLs use the
+        32-char hex form. The hyphenated form still resolves because the
+        converter's regex accepts both — Django reverses through ``to_url``
+        which always emits hex, so links in templates / changelists / inline
+        formsets all canonicalize automatically.
+
+        Non-UUID PKs (an ``IntegerField`` PK on some legacy table, for example)
+        won't match the converter's regex and fall back to the default
+        ``<path:object_id>`` patterns we still include after our swap.
+        """
+        info = self.opts.app_label, self.opts.model_name
+        object_routes = [
+            path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="%s_%s_history" % info),
+            path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="%s_%s_delete" % info),
+            path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="%s_%s_change" % info),
+        ]
+        # Append after super().get_urls() so our patterns are the
+        # *last-registered* ones with the canonical admin URL names — Django's
+        # reverse() picks the later registration when names collide, which is
+        # how we make ``reverse("admin:app_model_change", args=[obj.pk])``
+        # emit the hex form. The original ``<path:object_id>`` routes stay in
+        # place as a fallback for non-UUID PKs and for resolving inbound
+        # hyphenated URLs (the ``hexuuid`` regex accepts both forms anyway).
+        return super().get_urls() + object_routes

archivebox/base_models/models.py

@@ -2,11 +2,17 @@
 
 __package__ = "archivebox.base_models"
 
-from archivebox.uuid_compat import uuid7
+import json
+import shutil
+from typing import Any
+
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
 
 from django.db import models
 from django.db.models import F
+from django.db import transaction
+from django.db.models.signals import pre_delete
 from django.utils import timezone
 from django.contrib.auth import get_user_model
 from django.urls import reverse_lazy
@@ -15,6 +21,25 @@ from django.conf import settings
 from django_stubs_ext.db.models import TypedModelMeta
 
 
+def normalize_config_json_values(config: Any) -> Any:
+    if not isinstance(config, dict):
+        return config
+
+    normalized = dict(config)
+    for key, value in list(normalized.items()):
+        if not isinstance(value, str) or len(value) < 2:
+            continue
+        if value[:1] != '"' or value[-1:] != '"':
+            continue
+        try:
+            decoded = json.loads(value)
+        except ValueError:
+            continue
+        if isinstance(decoded, str):
+            normalized[key] = decoded
+    return normalized
+
+
 def get_or_create_system_user_pk(username="system"):
     User = get_user_model()
     # If there's exactly one superuser, use that for all system operations
@@ -32,7 +57,7 @@ class AutoDateTimeField(models.DateTimeField):
     """DateTimeField that automatically updates on save (legacy compatibility)."""
 
     def pre_save(self, model_instance, add):
-        if add or not getattr(model_instance, self.attname):
+        if add or self.attname not in model_instance.__dict__ or not model_instance.__dict__[self.attname]:
             value = timezone.now()
             setattr(model_instance, self.attname, value)
             return value
@@ -40,7 +65,7 @@ class AutoDateTimeField(models.DateTimeField):
 
 
 class ModelWithUUID(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
     created_by = models.ForeignKey(
@@ -96,7 +121,12 @@ class ModelWithHealthStats(models.Model):
     def increment_health_stats(self, success: bool):
         """Atomically increment success or failure counter using F() expression."""
         field = "num_uses_succeeded" if success else "num_uses_failed"
-        type(self).objects.filter(pk=self.pk).update(**{field: F(field) + 1})
+        type(self).objects.filter(pk=self.pk).update(
+            **{
+                field: F(field) + 1,
+                "modified_at": timezone.now(),
+            },
+        )
 
 
 class ModelWithConfig(models.Model):
@@ -107,14 +137,99 @@ class ModelWithConfig(models.Model):
     class Meta(TypedModelMeta):
         abstract = True
 
+    def save(self, *args, **kwargs):
+        normalized_config = normalize_config_json_values(self.config)
+        if normalized_config != self.config:
+            self.config = normalized_config
+            update_fields = kwargs.get("update_fields")
+            if update_fields is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
+        super().save(*args, **kwargs)
+
+
+class ModelWithDeleteAfter(models.Model):
+    delete_after_final_statuses: tuple[str, ...] = ()
+    delete_at = models.DateTimeField(default=None, null=True, blank=True, db_index=True)
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    def save(self, *args, **kwargs):
+        update_fields = kwargs.get("update_fields")
+        if self.delete_at is None:
+            self.set_delete_at_from_config()
+            if self.delete_at is not None and update_fields is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"]))
+        super().save(*args, **kwargs)
+
+    def get_delete_after_config_value(self):
+        from archivebox.config.common import get_config
+
+        return get_config(include_machine=False).DELETE_AFTER
+
+    def set_delete_at_from_config(self, config_value=None) -> bool:
+        if self.delete_at is not None:
+            return False
+
+        from archivebox.config.common import parse_delete_after
+
+        duration = parse_delete_after(self.get_delete_after_config_value() if config_value is None else config_value)
+        if duration is None:
+            return False
+
+        self.delete_at = (self.created_at or timezone.now()) + duration
+        return True
+
+    @classmethod
+    def missing_delete_at_candidates(cls):
+        return cls.objects.none()
+
+    @classmethod
+    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
+        if backfill_missing:
+            missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
+            for obj in missing_delete_at:
+                if obj.set_delete_at_from_config():
+                    cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
+                        delete_at=obj.delete_at,
+                        modified_at=timezone.now(),
+                    )
+
+        # Keep the expiration sweep anchored on delete_at. Some large tables
+        # have millions of final-status rows but almost no retained rows; adding
+        # the status filter before SQLite has narrowed by delete_at can make the
+        # runner scan the hot status index before it claims new work.
+        due_pks = list(
+            cls.objects.filter(delete_at__isnull=False, delete_at__lte=timezone.now())
+            .order_by("delete_at", "pk")
+            .values_list("pk", flat=True)[:batch_size],
+        )
+        if not due_pks:
+            return 0
+
+        queryset = cls.objects.filter(pk__in=due_pks)
+        if cls.delete_after_final_statuses:
+            queryset = queryset.filter(status__in=cls.delete_after_final_statuses)
+
+        count = 0
+        expired = list(queryset.order_by("delete_at", "pk"))
+        for obj in expired:
+            obj.delete()
+            count += 1
+        return count
+
 
 class ModelWithOutputDir(ModelWithUUID):
     class Meta(ModelWithUUID.Meta):
         abstract = True
 
+    _delete_signal_registered = False
+
     def save(self, *args, **kwargs):
         super().save(*args, **kwargs)
-        Path(self.output_dir).mkdir(parents=True, exist_ok=True)
+        output_dir = Path(self.output_dir)
+        # Avoid holding SQLite write transactions open across slow filesystem work.
+        transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
         # Note: index.json is deprecated, models should use write_index_jsonl() for full data
 
     @property
@@ -132,3 +247,53 @@ class ModelWithOutputDir(ModelWithUUID):
     @property
     def output_dir(self) -> Path:
         raise NotImplementedError(f"{self.__class__.__name__} must implement output_dir property")
+
+    def output_paths_for_delete(self) -> tuple[Path, ...]:
+        return (Path(self.output_dir),)
+
+    @classmethod
+    def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
+        from archivebox.config.common import get_config
+
+        data_dir = get_config().DATA_DIR.resolve()
+        safe_paths = []
+        for raw_path in paths:
+            path = Path(raw_path)
+            is_safe = False
+            for candidate in (path.absolute(), path.resolve()):
+                try:
+                    candidate.relative_to(data_dir)
+                    is_safe = True
+                    break
+                except ValueError:
+                    continue
+            if not is_safe:
+                raise ValueError(f"Refusing to delete output path outside DATA_DIR: {path}")
+            safe_paths.append(path)
+        return tuple(safe_paths)
+
+    @classmethod
+    def delete_output_paths(cls, paths) -> None:
+        for path in cls.validate_output_paths_for_delete(paths):
+            if path.is_symlink() or path.is_file():
+                path.unlink(missing_ok=True)
+            elif path.is_dir():
+                shutil.rmtree(path, ignore_errors=True)
+
+    @classmethod
+    def register_delete_signal(cls) -> None:
+        if cls._delete_signal_registered:
+            return
+
+        def schedule_output_dir_cleanup(sender, instance, **kwargs):
+            if not isinstance(instance, ModelWithOutputDir):
+                return
+            paths = instance.validate_output_paths_for_delete(instance.output_paths_for_delete())
+            transaction.on_commit(lambda paths=paths: instance.delete_output_paths(paths))
+
+        pre_delete.connect(
+            schedule_output_dir_cleanup,
+            dispatch_uid="archivebox.output_dir_cleanup_on_delete",
+            weak=False,
+        )
+        cls._delete_signal_registered = True

archivebox/cli/__init__.py

@@ -5,15 +5,25 @@ import sys
 from importlib import import_module
 
 import rich_click as click
-from rich import print
+from rich.console import Console
 
 from archivebox.config.version import VERSION
 
+STDERR = Console(stderr=True)
+
 
 if "--debug" in sys.argv:
     os.environ["DEBUG"] = "True"
     sys.argv.remove("--debug")
 
+# Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
+# `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
+# the subcommand executes. Strip it from argv here so each subcommand's own click parser
+# never sees it. Ignored for `help` and `init` themselves.
+if "--init" in sys.argv:
+    sys.argv = [arg for arg in sys.argv if arg != "--init"]
+    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+
 
 class ArchiveBoxGroup(click.Group):
     """lazy loading click group for archivebox commands"""
@@ -104,9 +114,8 @@ class ArchiveBoxGroup(click.Group):
         # handle renamed commands
         if cmd_name in self.renamed_commands:
             new_name = self.renamed_commands[cmd_name]
-            print(
+            STDERR.print(
                 f" [violet]Hint:[/violet] `archivebox {cmd_name}` has been renamed to `archivebox {new_name}`",
-                file=sys.stderr,
             )
             cmd_name = new_name
             ctx.invoked_subcommand = cmd_name
@@ -156,7 +165,8 @@ def cli(ctx, help=False):
 
     # if the subcommand is in archive_commands or model_commands,
     # then we need to set up the django environment and check that we're in a valid data folder
-    if subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands:
+    wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
+    if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
         # print('SETUP DJANGO AND CHECK DATA FOLDER')
         try:
             if subcommand == "server":
@@ -165,17 +175,26 @@ def cli(ctx, help=False):
                     os.environ["ARCHIVEBOX_RUNSERVER"] = "1"
                     if "--reload" in sys.argv:
                         os.environ["ARCHIVEBOX_AUTORELOAD"] = "1"
-                        from archivebox.config.common import get_config
-
-                        os.environ["ARCHIVEBOX_RUNSERVER_PIDFILE"] = str(get_config().TMP_DIR / "runserver.pid")
 
             from archivebox.config.django import setup_django
-            from archivebox.misc.checks import check_data_folder
+            from archivebox.misc.checks import check_data_folder, check_migrations
 
             setup_django()
+            if os.environ.get("ARCHIVEBOX_WANTS_INIT") == "1" and subcommand not in ("init", "help"):
+                # Universal `--init` was passed: build/upgrade the data folder before
+                # the regular preflight runs, so it succeeds on a fresh dir and an
+                # out-of-date schema both. Drop the env var afterwards so spawned
+                # subprocesses (supervisord workers, daphne, runner, etc.) inherit
+                # a clean env and don't re-trigger init in every child.
+                from archivebox.cli.archivebox_init import init as archivebox_init
+
+                archivebox_init(quick=True)
+                os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
             check_data_folder()
+            if subcommand != "update":
+                check_migrations(auto_apply=True)
         except Exception as e:
-            print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]", file=sys.stderr)
+            STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
             if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
                 raise
 
@@ -189,10 +208,29 @@ def main(args=None, prog_name=None, stdin=None):
     # stdin param allows passing input data from caller (used by __main__.py)
     # currently not used by click-based CLI, but kept for backwards compatibility
 
+    previous_unraisablehook = sys.unraisablehook
+
+    def ignore_shutdown_unraisable(unraisable):
+        if isinstance(unraisable.exc_value, (KeyboardInterrupt, SystemExit)):
+            return
+        previous_unraisablehook(unraisable)
+
+    sys.unraisablehook = ignore_shutdown_unraisable
     try:
-        cli(args=args, prog_name=prog_name)
+        cli(args=args, prog_name=prog_name, standalone_mode=False)
+    except click.Abort:
+        STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
+        raise SystemExit(130) from None
+    except click.ClickException as err:
+        err.show()
+        raise SystemExit(err.exit_code) from None
+    except click.exceptions.Exit as err:
+        raise SystemExit(err.exit_code) from None
     except KeyboardInterrupt:
-        print("\n\n[red][X] Got CTRL+C. Exiting...[/red]")
+        STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
+        raise SystemExit(130) from None
+    finally:
+        sys.unraisablehook = previous_unraisablehook
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_add.py

@@ -1,27 +1,24 @@
 #!/usr/bin/env python3
+from __future__ import annotations
 
 __package__ = "archivebox.cli"
 __command__ = "archivebox add"
 
 import sys
 import json
+import os
 from pathlib import Path
 
-from typing import TYPE_CHECKING
+from typing import Any, TYPE_CHECKING
 
 import rich_click as click
 
-from django.utils import timezone
-from django.db.models import QuerySet
-
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.util import parse_filesize_to_bytes
-from archivebox import CONSTANTS
-from archivebox.config.permissions import USER, HOSTNAME
-from archivebox.config.common import get_config
 
 
 if TYPE_CHECKING:
+    from django.db.models import QuerySet
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import Snapshot
 
@@ -52,6 +49,7 @@ def add(
     depth: int | str = 0,
     max_urls: int = 0,
     crawl_max_size: int | str = 0,
+    crawl_timeout: int = 0,
     snapshot_max_size: int | str = 0,
     crawl_max_concurrent_snapshots: int | None = None,
     tag: str = "",
@@ -60,12 +58,11 @@ def add(
     parser: str = "auto",
     plugins: str = "",
     persona: str = "Default",
-    overwrite: bool = False,
-    update: bool | None = None,
     index_only: bool = False,
     bg: bool = False,
     created_by_id: int | None = None,
-) -> tuple["Crawl", QuerySet["Snapshot"]]:
+    config: dict[str, Any] | None = None,
+) -> tuple[Crawl, QuerySet[Snapshot]]:
     """Add a new URL or list of URLs to your archive.
 
     The flow is:
@@ -83,10 +80,17 @@ def add(
     depth = int(depth)
     max_urls = int(max_urls or 0)
     crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
+    crawl_timeout = int(crawl_timeout or 0)
     snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
-    config = get_config()
+    from archivebox import CONSTANTS
+    from archivebox.config.permissions import USER, HOSTNAME
+    from archivebox.config.common import get_config
+
+    config_overrides = dict(config or {})
+    runtime_config = get_config()
+    crawl_max_concurrent_snapshots_override = crawl_max_concurrent_snapshots is not None
     if crawl_max_concurrent_snapshots is None:
-        crawl_max_concurrent_snapshots = config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
+        crawl_max_concurrent_snapshots = runtime_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
     crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)
 
     if depth not in (0, 1, 2, 3, 4):
@@ -95,6 +99,8 @@ def add(
         raise ValueError("max_urls must be >= 0")
     if crawl_max_size < 0:
         raise ValueError("crawl_max_size must be >= 0")
+    if crawl_timeout < 0:
+        raise ValueError("crawl_timeout must be >= 0")
     if snapshot_max_size < 0:
         raise ValueError("snapshot_max_size must be >= 0")
     if crawl_max_concurrent_snapshots < 1:
@@ -106,12 +112,13 @@ def add(
     from archivebox.personas.models import Persona
     from archivebox.misc.logging_util import printable_filesize
     from archivebox.misc.system import get_dir_size
-    from archivebox.services.runner import run_crawl
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    from django.contrib.auth import get_user_model
+    from django.utils import timezone
 
     created_by_id = created_by_id or get_or_create_system_user_pk()
+    created_by = get_user_model().objects.filter(pk=created_by_id).first()
     started_at = timezone.now()
-    if update is None:
-        update = not config.ONLY_NEW
 
     if isinstance(urls, str):
         url_list = [line.strip() for line in urls.splitlines() if line.strip()]
@@ -141,33 +148,45 @@ def add(
     # Read URLs directly into crawl
     urls_content = sources_file.read_text()
     persona_name = (persona or "Default").strip() or "Default"
-    plugins = plugins or str(config.get("PLUGINS") or "")
+    plugins = plugins or ""
     persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
     persona_obj.ensure_dirs()
+    effective_persona_config = get_config(persona=persona_obj, user=created_by)
+
+    crawl_config = {
+        "PERMISSIONS": str(effective_persona_config.PERMISSIONS),
+        **({"INDEX_ONLY": True} if index_only else {}),
+        **({"PLUGINS": plugins} if plugins else {}),
+        **(
+            {"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots}
+            if crawl_max_concurrent_snapshots_override
+            and crawl_max_concurrent_snapshots != int(effective_persona_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS)
+            else {}
+        ),
+        **({"CRAWL_MAX_URLS": max_urls} if max_urls else {}),
+        **({"CRAWL_MAX_SIZE": crawl_max_size} if crawl_max_size else {}),
+        **({"CRAWL_TIMEOUT": crawl_timeout} if crawl_timeout else {}),
+        **({"SNAPSHOT_MAX_SIZE": snapshot_max_size} if snapshot_max_size else {}),
+        **({"PARSER": parser} if parser != "auto" else {}),
+        **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
+        **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
+    }
+    # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
+    # priority — they win over persona/plugin/env defaults and get stamped
+    # directly onto crawl.config so the runtime resolution and admin UI both
+    # reflect them faithfully.
+    crawl_config.update(config_overrides)
 
     crawl = Crawl.objects.create(
         urls=urls_content,
         max_depth=depth,
-        max_urls=max_urls,
-        crawl_max_size=crawl_max_size,
-        snapshot_max_size=snapshot_max_size,
         tags_str=tag,
         persona_id=persona_obj.id,
         label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
         created_by_id=created_by_id,
-        status=Crawl.StatusChoices.QUEUED if bg or index_only else Crawl.StatusChoices.STARTED,
-        retry_at=timezone.now() if bg else None,
-        config={
-            "ONLY_NEW": not update,
-            "INDEX_ONLY": index_only,
-            "OVERWRITE": overwrite,
-            "PLUGINS": plugins,
-            "DEFAULT_PERSONA": persona_name,
-            "CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots,
-            "PARSER": parser,
-            **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
-            **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
-        },
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at=None if (index_only or bg) else timezone.now(),
+        config=crawl_config,
     )
 
     print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
@@ -179,13 +198,8 @@ def add(
     #    Discovered URLs become child Snapshots (depth+1)
 
     if index_only:
-        # Just create the crawl but don't start processing
-        print("[yellow]\\[*] Index-only mode - crawl created but not started[/yellow]")
-        crawl.create_snapshots_from_urls()
-        return crawl, crawl.snapshot_set.all()
-
-    if bg:
-        crawl.create_snapshots_from_urls()
+        print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
+        return crawl, crawl.snapshot_set.none()
 
     # 5. Start the crawl runner to process the queue
     #    The runner will:
@@ -195,25 +209,55 @@ def add(
     #    - Repeat until max_depth reached
 
     if bg:
-        # Background mode: just queue work and return (background runner via server will pick it up)
+        # Background mode: just queue work and return (background runner via server will pick it up).
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
-        run_crawl(str(crawl.id))
+        from archivebox.machine.models import Process
+        from archivebox.core.takeover_util import command_owns_runtime_stack, current_command, standby_until_runtime_stack_needed
+        from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
+
+        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
+        exit_code = 0
+        try:
+            try:
+                with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
+                    while True:
+                        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                        exit_code = run_runner_worker(
+                            ["--crawl-id", str(crawl.id)],
+                            name=f"worker_runner_add_{os.getpid()}",
+                            interactive_interrupts=True,
+                        )
+                        if exit_code == 0:
+                            break
+                        if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+                            continue
+                        raise SystemExit(exit_code)
+            except KeyboardInterrupt:
+                exit_code = 130
+                print("\n[red][X] archivebox add interrupted.[/red]")
+                print("[yellow]Hint: resume this crawl with:[/yellow]")
+                print(f"    [green]archivebox run --crawl-id={crawl.id}[/green]")
+                raise SystemExit(exit_code)
+        finally:
+            command.mark_exited(exit_code=exit_code)
+            stop_own_supervisord_process()
 
         # Print summary for foreground runs
         try:
             crawl.refresh_from_db()
-            snapshots_count = crawl.snapshot_set.count()
             try:
                 from django.db.models import Count, Sum
 
-                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("archiveresult__output_size"))
-                total_bytes = int(totals["total_bytes"] or 0) if totals["snapshot_count"] else 0
+                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("output_size"))
+                snapshots_count = int(totals["snapshot_count"] or 0)
+                total_bytes = int(totals["total_bytes"] or 0)
             except Exception:
+                snapshots_count = crawl.snapshot_set.count()
                 total_bytes, _, _ = get_dir_size(crawl.output_dir)
             total_size = printable_filesize(total_bytes)
             total_time = timezone.now() - started_at
@@ -234,12 +278,9 @@ def add(
             except Exception:
                 rel_output_str = str(crawl.output_dir)
 
-            bind_addr = config.BIND_ADDR or "127.0.0.1:8000"
-            if bind_addr.startswith("http://") or bind_addr.startswith("https://"):
-                base_url = bind_addr
-            else:
-                base_url = f"http://{bind_addr}"
-            admin_url = f"{base_url}/admin/crawls/crawl/{crawl.id}/change/"
+            from archivebox.core.routes_util import build_admin_url
+
+            admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
 
             print("\n[bold]crawl output saved to:[/bold]")
             print(f"  {rel_output_str}")
@@ -266,6 +307,7 @@ def add(
 )
 @click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)")
 @click.option("--crawl-max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
+@click.option("--crawl-timeout", type=int, default=0, help="Maximum total crawl runtime in seconds (0 = unlimited)")
 @click.option("--snapshot-max-size", default="0", help="Maximum per-snapshot size in bytes or units like 45mb / 1gb (0 = unlimited)")
 @click.option("--crawl-max-concurrent-snapshots", type=int, default=None, help="Maximum snapshots to archive concurrently within one crawl")
 @click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3")
@@ -274,33 +316,55 @@ def add(
 @click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
 @click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
 @click.option("--persona", default="Default", help="Authentication profile to use when archiving")
-@click.option("--overwrite", "-F", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
-@click.option("--update", is_flag=True, default=None, help="Retry any previously skipped/failed URLs when re-adding them")
+@click.option(
+    "--only-new/--no-only-new",
+    "only_new",
+    default=None,
+    help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
+    "Pass --no-only-new to force re-archive of URLs that already exist.",
+)
 @click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
+@click.option("--overwrite", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
+@click.option("--update", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
 @click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
 @click.argument("urls", nargs=-1, type=click.Path())
 @docstring(add.__doc__)
 def main(**kwargs):
     """Add a new URL or list of URLs to your archive"""
 
-    raw_urls = kwargs.pop("urls")
-    urls = _collect_input_urls(raw_urls)
-    if not urls:
-        raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
-    if int(kwargs.get("max_urls") or 0) < 0:
-        raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
-    try:
-        kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size"))
-    except ValueError as err:
-        raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err
-    try:
-        kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
-    except ValueError as err:
-        raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
-    if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
-        raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
 
-    add(urls=urls, **kwargs)
+    with foreground_shutdown_signals(), foreground_parent_watchdog():
+        raw_urls = kwargs.pop("urls")
+        urls = _collect_input_urls(raw_urls)
+        if not urls:
+            raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
+        if int(kwargs.get("max_urls") or 0) < 0:
+            raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
+        if int(kwargs.get("crawl_timeout") or 0) < 0:
+            raise click.BadParameter("crawl_timeout must be 0 or a positive integer.", param_hint="--crawl-timeout")
+        try:
+            kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size"))
+        except ValueError as err:
+            raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err
+        try:
+            kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
+        except ValueError as err:
+            raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
+        if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
+            raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
+
+        # Translate --only-new/--no-only-new into a crawl config override.
+        # add() takes config overrides as a dict; no per-flag kwargs.
+        overwrite = kwargs.pop("overwrite", False)
+        update = kwargs.pop("update", False)
+        only_new = kwargs.pop("only_new", None)
+        if overwrite or update:
+            only_new = False
+        if only_new is not None:
+            kwargs["config"] = {"ONLY_NEW": bool(only_new)}
+
+        add(urls=urls, **kwargs)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_archiveresult.py

@@ -38,7 +38,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict:
@@ -74,7 +74,7 @@ def create_archiveresults(
         1: Failure
     """
     from archivebox.config.common import get_config
-    from archivebox.hooks import discover_hooks
+    from archivebox.plugins.hooks import discover_hooks
     from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
     from archivebox.core.models import Snapshot
 
@@ -152,7 +152,7 @@ def create_archiveresults(
             config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
             hooks = discover_hooks("Snapshot", config=config)
             for hook_path in hooks:
-                hook_name = hook_path.name
+                hook_name = hook_path.stem
                 plugin_name = hook_path.parent.name
                 if not is_tty:
                     write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))

archivebox/cli/archivebox_binary.py

@@ -33,7 +33,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================

archivebox/cli/archivebox_config.py

@@ -31,7 +31,7 @@ def config(
     from abx_plugins.plugins.base.utils import resolve_alias
     from archivebox.config.collection import write_config_file
     from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
 
     check_data_folder()
 

archivebox/cli/archivebox_crawl.py

@@ -39,7 +39,7 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================
@@ -247,17 +247,30 @@ def update_crawls(
         try:
             crawl = Crawl.objects.get(id=crawl_id)
 
-            # Apply updates from CLI flags
             if status:
-                crawl.status = status
-                crawl.retry_at = timezone.now()
+                if status not in Crawl.StatusChoices.values:
+                    rprint(f"[red]Invalid crawl status: {status}[/red]", file=sys.stderr)
+                    continue
+                if status == Crawl.StatusChoices.SEALED:
+                    crawl.cancel()
+                elif status == Crawl.StatusChoices.PAUSED:
+                    crawl.pause()
+                elif status == Crawl.StatusChoices.QUEUED:
+                    if crawl.status == Crawl.StatusChoices.PAUSED:
+                        crawl.resume()
+                    else:
+                        crawl.update_and_requeue(status=Crawl.StatusChoices.QUEUED, retry_at=timezone.now())
+                elif status == Crawl.StatusChoices.STARTED:
+                    crawl.update_and_requeue(status=Crawl.StatusChoices.STARTED, retry_at=timezone.now())
             if max_depth is not None:
+                crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
                 crawl.max_depth = max_depth
-
-            crawl.save()
+            elif not status:
+                crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
             updated_count += 1
 
             if not is_tty:
+                crawl.refresh_from_db()
                 write_record(crawl.to_json())
 
         except Crawl.DoesNotExist:

archivebox/cli/archivebox_crawl_compat.py

@@ -19,7 +19,7 @@ from archivebox.cli.archivebox_add import add
 def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]):
     """Backwards-compatible `archivebox crawl URL...` entrypoint."""
     del status, wait
-    add(list(urls), depth=depth, tag=tag, index_only=True, bg=True)
+    add(list(urls), depth=depth, tag=tag, bg=True)
     sys.exit(0)
 
 

archivebox/cli/archivebox_extract.py

@@ -32,6 +32,7 @@ __command__ = "archivebox extract"
 
 import sys
 from collections import defaultdict
+from itertools import product
 
 import rich_click as click
 
@@ -58,19 +59,35 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
 
     try:
+        was_paused = archiveresult.snapshot.is_paused
         archiveresult.reset_for_retry()
         snapshot = archiveresult.snapshot
-        snapshot.status = snapshot.StatusChoices.QUEUED
-        snapshot.retry_at = timezone.now()
-        snapshot.save(update_fields=["status", "retry_at", "modified_at"])
-
+        if not was_paused:
+            snapshot.queue_for_extraction()
+        else:
+            # A paused snapshot may still accept explicit maintenance for one
+            # ArchiveResult, but this path must not transition it back to
+            # queued/startable work. Guard: only set retry_at while the row is
+            # still paused — concurrent resume would otherwise see a stale
+            # retry_at marker.
+            snapshot.safe_update(
+                {"retry_at": timezone.now()},
+                refresh=False,
+                extra_filter={"status": snapshot.StatusChoices.PAUSED},
+            )
         crawl = snapshot.crawl
-        if crawl.status != crawl.StatusChoices.STARTED:
-            crawl.status = crawl.StatusChoices.QUEUED
-        crawl.retry_at = timezone.now()
-        crawl.save(update_fields=["status", "retry_at", "modified_at"])
+        if not crawl.claim_processing_lock(lock_seconds=10):
+            rprint(
+                f"[yellow]Crawl {crawl.id} is already owned by another runner[/yellow]",
+                file=sys.stderr,
+            )
+            return 1
 
-        run_crawl(str(crawl.id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
+        try:
+            run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
+        finally:
+            if was_paused:
+                snapshot.restore_paused_scheduler_marker()
         archiveresult.refresh_from_db()
 
         if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
@@ -98,6 +115,7 @@ def run_plugins(
     plugins: str = "",
     wait: bool = True,
     emit_results: bool = True,
+    show_progress: bool = True,
 ) -> int:
     """
     Run plugins on Snapshots from input.
@@ -118,7 +136,9 @@ def run_plugins(
         TYPE_ARCHIVERESULT,
     )
     from archivebox.core.models import Snapshot
+    from archivebox.core.models import ArchiveResult
     from archivebox.services.runner import run_crawl
+    from abx_dl.models import discover_plugins
 
     is_tty = sys.stdout.isatty()
 
@@ -145,7 +165,7 @@ def run_plugins(
         if record_type == TYPE_SNAPSHOT:
             snapshot_id = record.get("id")
             if snapshot_id:
-                snapshot_ids.add(snapshot_id)
+                snapshot_ids.add(str(snapshot_id))
             elif record.get("url"):
                 # Look up by URL (get most recent if multiple exist)
                 snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first()
@@ -157,60 +177,162 @@ def run_plugins(
         elif record_type == TYPE_ARCHIVERESULT:
             snapshot_id = record.get("snapshot_id")
             if snapshot_id:
-                snapshot_ids.add(snapshot_id)
+                snapshot_ids.add(str(snapshot_id))
                 plugin_name = record.get("plugin")
                 if plugin_name and not plugins_list:
                     requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name))
 
         elif "id" in record:
             # Assume it's a snapshot ID
-            snapshot_ids.add(record["id"])
+            snapshot_ids.add(str(record["id"]))
 
     if not snapshot_ids:
         rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr)
         return 1
 
-    # Get snapshots and ensure they have pending ArchiveResults
-    processed_count = 0
-    for snapshot_id in snapshot_ids:
-        try:
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-        except Snapshot.DoesNotExist:
-            rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
-            continue
+    existing_snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids).values_list("id", "crawl_id"))
+    existing_snapshot_ids = {str(snapshot_id) for snapshot_id, _crawl_id in existing_snapshots}
+    existing_crawl_ids = {str(crawl_id) for _snapshot_id, crawl_id in existing_snapshots}
+    missing_snapshot_ids = sorted(str(snapshot_id) for snapshot_id in snapshot_ids - existing_snapshot_ids)
+    for snapshot_id in missing_snapshot_ids:
+        rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
 
-        requested_plugin_names = set(plugins_list) | requested_plugins_by_snapshot.get(str(snapshot.id), set())
-        for plugin_name in requested_plugin_names:
-            existing_result = snapshot.archiveresult_set.filter(plugin=plugin_name).order_by("-created_at").first()
-            if existing_result:
-                existing_result.reset_for_retry()
+    # Queue only the target plugin rows. Bulk updates keep large reindex runs
+    # from doing one SELECT+UPDATE per snapshot/plugin before hooks even start.
+    requested_pairs: set[tuple[str, str]] = set()
+    if plugins_list:
+        requested_pairs.update((snapshot_id, plugin_name) for snapshot_id, plugin_name in product(existing_snapshot_ids, plugins_list))
+    else:
+        requested_pairs.update(
+            (snapshot_id, plugin_name)
+            for snapshot_id, plugin_names in requested_plugins_by_snapshot.items()
+            if snapshot_id in existing_snapshot_ids
+            for plugin_name in plugin_names
+        )
+    plugins_by_name = discover_plugins()
+    requested_rows: set[tuple[str, str, str]] = set()
+    for snapshot_id, plugin_name in requested_pairs:
+        plugin = plugins_by_name.get(plugin_name)
+        hooks = plugin.filter_hooks("Snapshot") if plugin is not None else []
+        if hooks:
+            requested_rows.update((snapshot_id, plugin_name, hook.name) for hook in hooks)
+        else:
+            requested_rows.add((snapshot_id, plugin_name, ""))
 
-        # Reset snapshot status to allow processing
-        if snapshot.status == Snapshot.StatusChoices.SEALED:
-            snapshot.status = Snapshot.StatusChoices.STARTED
-            snapshot.retry_at = timezone.now()
-            snapshot.save()
+    reset_fields = {
+        "status": ArchiveResult.StatusChoices.QUEUED,
+        "output_str": "",
+        "output_json": None,
+        "output_files": {},
+        "output_size": 0,
+        "output_mimetypes": "",
+        "start_ts": None,
+        "end_ts": None,
+        "modified_at": timezone.now(),
+    }
+    if plugins_list:
+        ArchiveResult.objects.filter(snapshot_id__in=existing_snapshot_ids, plugin__in=plugins_list).update(**reset_fields)
+    elif requested_plugins_by_snapshot:
+        snapshot_ids_by_plugin: dict[str, set[str]] = defaultdict(set)
+        for snapshot_id, plugin_names in requested_plugins_by_snapshot.items():
+            if snapshot_id in existing_snapshot_ids:
+                for plugin_name in plugin_names:
+                    snapshot_ids_by_plugin[plugin_name].add(snapshot_id)
+        for plugin_name, plugin_snapshot_ids in snapshot_ids_by_plugin.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(**reset_fields)
+    existing_rows = set(
+        ArchiveResult.objects.filter(
+            snapshot_id__in=existing_snapshot_ids,
+            plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
+        ).values_list("snapshot_id", "plugin", "hook_name"),
+    )
+    missing_rows = requested_rows - {(str(snapshot_id), plugin_name, hook_name) for snapshot_id, plugin_name, hook_name in existing_rows}
+    if missing_rows:
+        ArchiveResult.objects.bulk_create(
+            [
+                ArchiveResult(
+                    snapshot_id=snapshot_id,
+                    plugin=plugin_name,
+                    hook_name=hook_name,
+                    status=ArchiveResult.StatusChoices.QUEUED,
+                )
+                for snapshot_id, plugin_name, hook_name in sorted(missing_rows)
+            ],
+            batch_size=500,
+        )
 
-        processed_count += 1
+    processed_count = len(existing_snapshot_ids)
+    queue_at = timezone.now()
+    if existing_snapshot_ids:
+        if requested_rows:
+            # Targeted ArchiveResult retries use retry_at as the scheduling
+            # signal and keep sealed snapshots sealed so extractors are not
+            # re-run outside the explicitly queued plugin rows. Paused snapshots
+            # also keep status=paused here: `retry_at` only asks the orchestrator
+            # to process the queued plugin rows, and run_due_snapshot restores
+            # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
+            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "modified_at"):
+                # Guard the read-time status so we never bump retry_at on a
+                # row that's been re-queued / started by a concurrent runner.
+                snapshot.safe_update(
+                    {"retry_at": queue_at, "modified_at": queue_at},
+                    refresh=False,
+                    extra_filter={"status": snapshot.status},
+                )
+        else:
+            # No plugin rows were requested, so this is a full snapshot retry.
+            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):
+                snapshot.safe_update(
+                    {
+                        "status": Snapshot.StatusChoices.QUEUED,
+                        "retry_at": queue_at,
+                        "current_step": 0,
+                        "modified_at": queue_at,
+                    },
+                    refresh=False,
+                    extra_filter={"status": snapshot.status},
+                )
+    if existing_crawl_ids and not requested_rows:
+        from archivebox.crawls.models import Crawl
+
+        for crawl in Crawl.objects.filter(id__in=existing_crawl_ids).only("id", "status", "retry_at", "modified_at"):
+            update_fields = {
+                "retry_at": queue_at,
+                "modified_at": queue_at,
+            }
+            if crawl.status != Crawl.StatusChoices.STARTED:
+                update_fields["status"] = Crawl.StatusChoices.QUEUED
+            crawl.safe_update(
+                update_fields,
+                refresh=False,
+                extra_filter={"status": crawl.status},
+            )
 
     if processed_count == 0:
         rprint("[red]No snapshots to process[/red]", file=sys.stderr)
         return 1
 
-    rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
+    if show_progress:
+        rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
 
     # Run orchestrator if --wait (default)
     if wait:
-        rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
+        if show_progress:
+            rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
         snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id in snapshot_ids:
-            try:
-                snapshot = Snapshot.objects.only("id", "crawl_id").get(id=snapshot_id)
-            except Snapshot.DoesNotExist:
-                continue
-            snapshot_ids_by_crawl[str(snapshot.crawl_id)].add(str(snapshot.id))
+        for snapshot_id, crawl_id in existing_snapshots:
+            snapshot_ids_by_crawl[str(crawl_id)].add(str(snapshot_id))
 
         for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items():
+            from archivebox.crawls.models import Crawl
+
+            crawl = Crawl.objects.get(id=crawl_id)
+            if not crawl.claim_processing_lock(lock_seconds=10):
+                rprint(
+                    f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]",
+                    file=sys.stderr,
+                )
+                return 1
             selected_plugins = (
                 plugins_list
                 or sorted(
@@ -222,6 +344,7 @@ def run_plugins(
                 crawl_id,
                 snapshot_ids=sorted(crawl_snapshot_ids),
                 selected_plugins=selected_plugins,
+                show_progress=show_progress,
             )
 
     if not emit_results:

archivebox/cli/archivebox_help.py

@@ -2,6 +2,8 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox help"
 
+import ast
+import importlib.util
 import os
 from pathlib import Path
 
@@ -10,6 +12,25 @@ from rich import print
 from rich.panel import Panel
 
 
+def _command_doc(cmd: str, import_path: str) -> str:
+    modname, _ = import_path.rsplit(".", 1)
+    spec = importlib.util.find_spec(modname)
+    if spec is None or spec.origin is None:
+        return ""
+
+    try:
+        tree = ast.parse(Path(spec.origin).read_text())
+    except OSError:
+        return ""
+
+    for name in (cmd, "main"):
+        for node in tree.body:
+            if isinstance(node, ast.FunctionDef) and node.name == name:
+                return (ast.get_docstring(node) or "").splitlines()[0]
+
+    return (ast.get_docstring(tree) or "").splitlines()[0]
+
+
 def help() -> None:
     """Print the ArchiveBox help message and usage"""
 
@@ -22,16 +43,12 @@ def help() -> None:
     log_cli_command("help", [], None, ".")
 
     COMMANDS_HELP_TEXT = (
-        "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.meta_commands.keys()
-        )
+        "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.meta_commands.items())
+        + "\n\n    "
+        + "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.setup_commands.items())
         + "\n\n    "
         + "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.setup_commands.keys()
-        )
-        + "\n\n    "
-        + "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.archive_commands.keys()
+            f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.archive_commands.items()
         )
     )
 

archivebox/cli/archivebox_init.py

@@ -29,6 +29,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     from archivebox.config.common import get_config
     from archivebox.config.collection import write_config_file
     from archivebox.misc.db import apply_migrations
+    from archivebox.misc.checks import check_migrations
 
     config = get_config()
 
@@ -72,6 +73,8 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     config.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
     config.USERS_DIR.mkdir(parents=True, exist_ok=True)
     Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
+    for path in (Path(CONSTANTS.SOURCES_DIR), config.ARCHIVE_DIR, config.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
+        path.chmod(int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)
 
     print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, DATA_DIR)}...")
 
@@ -91,6 +94,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     from archivebox.config.django import setup_django
 
     setup_django()
+    check_migrations(blocking=True, auto_apply=False)
 
     for migration_line in apply_migrations(DATA_DIR):
         sys.stdout.write(f"    {migration_line}\n")

archivebox/cli/archivebox_install.py

@@ -28,6 +28,11 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
 
     config = get_config()
     archive_dir = config.ARCHIVE_DIR
+
+    if dry_run:
+        print("[dim]Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow[/dim]")
+        return
+
     if not (os.access(archive_dir, os.R_OK) and archive_dir.is_dir()):
         init()  # must init full index because we need a db to store Binary entries in
 
@@ -47,10 +52,6 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
         print(f"    DATA_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].")
         print()
 
-    if dry_run:
-        print("[dim]Dry run - would run the abx-dl install flow[/dim]")
-        return
-
     # Set up Django
     from archivebox.config.django import setup_django
 

archivebox/cli/archivebox_list.py

@@ -20,7 +20,7 @@ from archivebox.cli.archivebox_snapshot import list_snapshots
 @click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
 @click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
 @click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
+@click.option("--search", help="Search mode to use for the query")
 @click.argument("query", nargs=-1)
 def main(
     status: str | None,

archivebox/cli/archivebox_machine.py

@@ -27,7 +27,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================

archivebox/cli/archivebox_persona.py

@@ -31,17 +31,13 @@ import os
 import sys
 import shutil
 import platform
-import subprocess
-import tempfile
-import json
 from pathlib import Path
 from collections.abc import Iterable
-from collections import OrderedDict
 
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 from archivebox.personas import importers as persona_importers
 
 
@@ -214,136 +210,6 @@ CHROMIUM_BROWSERS = {"chrome", "chromium", "brave", "edge"}
 # Cookie Extraction via CDP
 # =============================================================================
 
-NETSCAPE_COOKIE_HEADER = [
-    "# Netscape HTTP Cookie File",
-    "# https://curl.se/docs/http-cookies.html",
-    "# This file was generated by ArchiveBox persona cookie extraction",
-    "#",
-    "# Format: domain\\tincludeSubdomains\\tpath\\tsecure\\texpiry\\tname\\tvalue",
-    "",
-]
-
-
-def _parse_netscape_cookies(path: Path) -> "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]":
-    cookies = OrderedDict()
-    if not path.exists():
-        return cookies
-
-    for line in path.read_text().splitlines():
-        if not line or line.startswith("#"):
-            continue
-        parts = line.split("\t")
-        if len(parts) < 7:
-            continue
-        domain, include_subdomains, cookie_path, secure, expiry, name, value = parts[:7]
-        key = (domain, cookie_path, name)
-        cookies[key] = (domain, include_subdomains, cookie_path, secure, expiry, name, value)
-    return cookies
-
-
-def _write_netscape_cookies(path: Path, cookies: "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]") -> None:
-    lines = list(NETSCAPE_COOKIE_HEADER)
-    for cookie in cookies.values():
-        lines.append("\t".join(cookie))
-    path.write_text("\n".join(lines) + "\n")
-
-
-def _merge_netscape_cookies(existing_file: Path, new_file: Path) -> None:
-    existing = _parse_netscape_cookies(existing_file)
-    new = _parse_netscape_cookies(new_file)
-    for key, cookie in new.items():
-        existing[key] = cookie
-    _write_netscape_cookies(existing_file, existing)
-
-
-def extract_cookies_via_cdp(
-    user_data_dir: Path,
-    output_file: Path,
-    profile_dir: str | None = None,
-    chrome_binary: str | None = None,
-) -> bool:
-    """
-    Launch Chrome with the given user data dir and extract cookies via CDP.
-
-    Returns True if successful, False otherwise.
-    """
-    from archivebox.config.common import get_config
-
-    # Find the cookie extraction script
-    chrome_plugin_dir = Path(__file__).parent.parent / "plugins" / "chrome"
-    extract_script = chrome_plugin_dir / "extract_cookies.js"
-
-    if not extract_script.exists():
-        rprint(f"[yellow]Cookie extraction script not found at {extract_script}[/yellow]", file=sys.stderr)
-        return False
-
-    # Get node modules dir
-    node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules"
-
-    # Set up environment
-    env = os.environ.copy()
-    env["NODE_MODULES_DIR"] = str(node_modules_dir)
-    env["CHROME_USER_DATA_DIR"] = str(user_data_dir)
-    env["CHROME_HEADLESS"] = "true"
-    if chrome_binary:
-        env["CHROME_BINARY"] = str(chrome_binary)
-    output_path = output_file
-    temp_output = None
-    temp_dir = None
-    if output_file.exists():
-        temp_dir = Path(tempfile.mkdtemp(prefix="ab_cookies_"))
-        temp_output = temp_dir / "cookies.txt"
-        output_path = temp_output
-    if profile_dir:
-        extra_arg = f"--profile-directory={profile_dir}"
-        existing_extra = env.get("CHROME_ARGS_EXTRA", "").strip()
-        args_list = []
-        if existing_extra:
-            if existing_extra.startswith("["):
-                try:
-                    parsed = json.loads(existing_extra)
-                    if isinstance(parsed, list):
-                        args_list.extend(str(x) for x in parsed)
-                except Exception:
-                    args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()])
-            else:
-                args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()])
-        args_list.append(extra_arg)
-        env["CHROME_ARGS_EXTRA"] = json.dumps(args_list)
-
-    env["COOKIES_OUTPUT_FILE"] = str(output_path)
-
-    try:
-        result = subprocess.run(
-            ["node", str(extract_script)],
-            env=env,
-            capture_output=True,
-            text=True,
-            timeout=60,
-        )
-
-        if result.returncode == 0:
-            if temp_output and temp_output.exists():
-                _merge_netscape_cookies(output_file, temp_output)
-            return True
-        else:
-            rprint(f"[yellow]Cookie extraction failed: {result.stderr}[/yellow]", file=sys.stderr)
-            return False
-
-    except subprocess.TimeoutExpired:
-        rprint("[yellow]Cookie extraction timed out[/yellow]", file=sys.stderr)
-        return False
-    except FileNotFoundError:
-        rprint("[yellow]Node.js not found. Cannot extract cookies.[/yellow]", file=sys.stderr)
-        return False
-    except Exception as e:
-        rprint(f"[yellow]Cookie extraction error: {e}[/yellow]", file=sys.stderr)
-        return False
-    finally:
-        if temp_dir and temp_dir.exists():
-            shutil.rmtree(temp_dir, ignore_errors=True)
-
-
 # =============================================================================
 # Validation Helpers
 # =============================================================================

archivebox/cli/archivebox_pluginmap.py

@@ -16,8 +16,9 @@ EVENT_FLOW_DIAGRAM = """
 │  InstallEvent                                                               │
 │    └─ config.json > required_binaries                                       │
 │         └─ BinaryRequestEvent                                               │
-│              └─ on_BinaryRequest__*                                         │
+│              └─ abxpkg BinaryService builtin providers                      │
 │                   └─ BinaryEvent                                            │
+│                        └─ BinaryCacheService / project cache backend        │
 │                                                                             │
 │  CrawlEvent                                                                 │
 │    └─ CrawlSetupEvent                                                       │
@@ -56,13 +57,15 @@ def pluginmap(
     from rich.panel import Panel
     from rich import box
 
-    from archivebox.hooks import (
-        BUILTIN_PLUGINS_DIR,
-        USER_PLUGINS_DIR,
+    from archivebox.plugins.hooks import (
         discover_hooks,
         is_background_hook,
         normalize_hook_event_name,
     )
+    from archivebox.plugins.discovery import (
+        BUILTIN_PLUGINS_DIR,
+        USER_PLUGINS_DIR,
+    )
 
     console = Console()
     prnt = console.print
@@ -73,12 +76,14 @@ def pluginmap(
             "emits": ["BinaryRequestEvent", "BinaryEvent", "ProcessEvent"],
         },
         "BinaryRequestEvent": {
-            "description": "Provider phase. on_BinaryRequest hooks resolve or install requested binaries.",
+            "description": "Binary resolution phase. abxpkg BinaryService resolves or installs requested binaries using built-in providers.",
             "emits": ["BinaryEvent", "ProcessEvent"],
+            "direct_hooks": False,
         },
         "BinaryEvent": {
-            "description": "Resolved binary metadata event. Projected into the DB binary cache.",
+            "description": "Resolved binary metadata event. ArchiveBoxBinaryService projects it into the ArchiveBox DB binary cache.",
             "emits": [],
+            "direct_hooks": False,
         },
         "CrawlEvent": {
             "description": "Root crawl lifecycle event emitted by the runner.",
@@ -138,7 +143,8 @@ def pluginmap(
 
     for event_name, info in event_phases.items():
         hook_event = normalize_hook_event_name(event_name)
-        hooks = discover_hooks(event_name, filter_disabled=not show_disabled)
+        has_direct_hooks = info.get("direct_hooks", True)
+        hooks = discover_hooks(event_name, filter_disabled=not show_disabled) if has_direct_hooks else []
 
         hook_infos = []
         for hook_path in hooks:
@@ -192,8 +198,10 @@ def pluginmap(
         prnt(f"[dim]{info['description']}[/dim]")
         if info["emits"]:
             prnt(f"[dim]Emits: {', '.join(info['emits'])}[/dim]")
-        if not hook_infos:
+        if not hook_infos and has_direct_hooks:
             prnt(f"[dim]No direct on_{hook_event}__* scripts are currently defined for this event family.[/dim]")
+        elif not has_direct_hooks:
+            prnt("[dim]No direct plugin hook family. This event is handled by services.[/dim]")
         prnt()
 
     if not quiet:

archivebox/cli/archivebox_process.py

@@ -30,7 +30,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================
@@ -54,7 +54,7 @@ def list_processes(
 
     is_tty = sys.stdout.isatty()
 
-    queryset = Process.objects.all().select_related("binary", "machine").order_by("-start_ts")
+    queryset = Process.objects.all().select_related("binary", "machine").order_by("-started_at", "-created_at")
 
     # Apply filters
     filter_kwargs = {}

archivebox/cli/archivebox_remove.py

@@ -3,16 +3,16 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox remove"
 
-import shutil
+import time
 from pathlib import Path
 from collections.abc import Iterable
 
 import rich_click as click
 
+from django.db import OperationalError
 from django.db.models import QuerySet
 
 from archivebox.config import DATA_DIR
-from archivebox.config.common import get_config
 from archivebox.config.django import setup_django
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.checks import check_data_folder
@@ -33,7 +33,6 @@ def remove(
     after: float | None = None,
     before: float | None = None,
     yes: bool = False,
-    delete: bool = False,
     out_dir: Path = DATA_DIR,
 ) -> QuerySet:
     """Remove the specified URLs from the archive"""
@@ -63,26 +62,46 @@ def remove(
         raise SystemExit(1)
 
     log_list_finished(snapshots)
-    log_removal_started(snapshots, yes=yes, delete=delete)
+    log_removal_started(snapshots, yes=yes)
 
-    timer = TimedProgress(360, prefix="      ")
-    try:
-        for snapshot in snapshots:
-            if delete:
-                shutil.rmtree(snapshot.output_dir, ignore_errors=True)
-                legacy_path = get_config().ARCHIVE_DIR / snapshot.timestamp
-                if legacy_path.is_symlink():
-                    legacy_path.unlink(missing_ok=True)
-    finally:
-        timer.end()
-
-    to_remove = snapshots.count()
-
-    from archivebox.search import flush_search_index
     from archivebox.core.models import Snapshot
+    from archivebox.search.query import flush_search_index
+
+    # Freeze the target set up-front so a concurrent daemon writing new
+    # snapshots can't extend the deletion under us, and so the cursor isn't
+    # held open across the per-row deletes below.
+    snapshot_pks = list(snapshots.values_list("pk", flat=True))
+    to_remove = len(snapshot_pks)
+
+    # Search-index flush touches a separate backend (FTS / sonic), not the
+    # main index.sqlite3 writer lock, so it's safe to do once up front.
+    flush_search_index(snapshots=Snapshot.objects.filter(pk__in=snapshot_pks))
+
+    # Delete one snapshot at a time. Each ``.delete()`` is its own short
+    # Django-atomic block, so the writer lock is released between rows and
+    # an in-flight daemon transaction can interleave instead of deadlocking.
+    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
+    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
+    # rmtree doesn't hold the lock either.
+    #
+    # The SQLite retry wrapper in core/sqlite_backend/base.py re-raises lock
+    # errors when called inside an atomic block (because it can't safely
+    # release+reacquire a transaction), so we wrap each row's delete in our
+    # own retry loop at this outer (non-atomic) level. Each attempt is a
+    # fresh atomic; an exception cleanly rolls it back before we sleep.
+    retry_interval = 1.0
+    retry_timeout = 60.0
+    for pk in snapshot_pks:
+        deadline = time.monotonic() + retry_timeout
+        while True:
+            try:
+                Snapshot.objects.filter(pk=pk).delete()
+                break
+            except OperationalError as err:
+                if "database is locked" not in str(err) or time.monotonic() >= deadline:
+                    raise
+                time.sleep(retry_interval)
 
-    flush_search_index(snapshots=snapshots)
-    snapshots.delete()
     all_snapshots = Snapshot.objects.all()
     log_removal_finished(all_snapshots.count(), to_remove)
 
@@ -91,7 +110,6 @@ def remove(
 
 @click.command()
 @click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
-@click.option("--delete", is_flag=True, help="Delete the archived content and metadata folder in addition to removing from index")
 @click.option("--before", type=float, help="Remove only URLs bookmarked before timestamp")
 @click.option("--after", type=float, help="Remove only URLs bookmarked after timestamp")
 @click.option(

archivebox/cli/archivebox_run.py

@@ -41,6 +41,7 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox run"
 
 import sys
+import asyncio
 from collections import defaultdict
 
 import rich_click as click
@@ -76,6 +77,7 @@ def process_stdin_records() -> int:
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot, ArchiveResult
     from archivebox.crawls.models import Crawl
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Binary
     from archivebox.services.runner import run_binary, run_crawl
 
@@ -111,10 +113,10 @@ def process_stdin_records() -> int:
                     crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if crawl:
-                    crawl.retry_at = timezone.now()
-                    if crawl.status not in [Crawl.StatusChoices.SEALED]:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save()
+                    crawl.update_and_requeue(
+                        status=Crawl.StatusChoices.QUEUED,
+                        retry_at=timezone.now(),
+                    )
                     full_crawl_ids.add(str(crawl.id))
                     run_all_plugins_for_crawl.add(str(crawl.id))
                     output_records.append(crawl.to_json())
@@ -132,15 +134,7 @@ def process_stdin_records() -> int:
                     snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if snapshot:
-                    snapshot.retry_at = timezone.now()
-                    if snapshot.status not in [Snapshot.StatusChoices.SEALED]:
-                        snapshot.status = Snapshot.StatusChoices.QUEUED
-                    snapshot.save()
-                    crawl = snapshot.crawl
-                    crawl.retry_at = timezone.now()
-                    if crawl.status != Crawl.StatusChoices.STARTED:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save(update_fields=["status", "retry_at", "modified_at"])
+                    snapshot.queue_for_extraction()
                     crawl_id = str(snapshot.crawl_id)
                     snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
                     run_all_plugins_for_crawl.add(crawl_id)
@@ -177,15 +171,7 @@ def process_stdin_records() -> int:
                         snapshot = None
 
                 if snapshot:
-                    snapshot.retry_at = timezone.now()
-                    if snapshot.status != Snapshot.StatusChoices.STARTED:
-                        snapshot.status = Snapshot.StatusChoices.QUEUED
-                    snapshot.save(update_fields=["status", "retry_at", "modified_at"])
-                    crawl = snapshot.crawl
-                    crawl.retry_at = timezone.now()
-                    if crawl.status != Crawl.StatusChoices.STARTED:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save(update_fields=["status", "retry_at", "modified_at"])
+                    snapshot.queue_for_extraction()
                     crawl_id = str(snapshot.crawl_id)
                     snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
                     if plugin_name:
@@ -236,15 +222,23 @@ def process_stdin_records() -> int:
     targeted_crawl_ids = full_crawl_ids | set(snapshot_ids_by_crawl)
     if targeted_crawl_ids:
         for crawl_id in sorted(targeted_crawl_ids):
-            run_crawl(
-                crawl_id,
-                snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
-                selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
-            )
+            try:
+                crawl = Crawl.objects.get(id=crawl_id)
+            except Crawl.DoesNotExist:
+                continue
+            if not crawl.claim_processing_lock(lock_seconds=10):
+                rprint(f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]", file=sys.stderr)
+                return 1
+            with foreground_shutdown_signals(), foreground_parent_watchdog():
+                run_crawl(
+                    crawl_id,
+                    snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
+                    selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
+                )
     return 0
 
 
-def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int:
+def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int:
     """
     Run the background runner loop.
 
@@ -253,25 +247,50 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int:
 
     Returns exit code (0 = success, 1 = error).
     """
-    from django.utils import timezone
+    from archivebox.config import CONSTANTS
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Machine, Process
-    from archivebox.services.runner import recover_orphaned_crawls, recover_orphaned_snapshots, run_pending_crawls
+    from archivebox.core.takeover_util import enter_single_runner_gate
+    from archivebox.services.runner import recover_orchestrator_state, run_pending_crawls
 
-    Process.cleanup_stale_running()
-    Process.cleanup_orphaned_workers()
-    Process.cleanup_orphaned_chrome()
-    recover_orphaned_snapshots()
-    recover_orphaned_crawls()
     Machine.current()
     current = Process.current()
-    if current.process_type != Process.TypeChoices.ORCHESTRATOR:
-        current.process_type = Process.TypeChoices.ORCHESTRATOR
-        current.save(update_fields=["process_type", "modified_at"])
-
-    try:
-        run_pending_crawls(daemon=daemon, crawl_id=crawl_id)
+    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
+        current.mark_exited()
         return 0
-    except KeyboardInterrupt:
+
+    recover_orchestrator_state(include_chrome=True)
+    if crawl_id:
+        from django.utils import timezone
+        from archivebox.crawls.models import Crawl
+
+        crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
+        now = timezone.now()
+        # Only re-lease when the row is unscheduled (retry_at IS NULL) or its
+        # existing lease has already expired. A future retry_at means another
+        # worker is already scheduled — don't clobber.
+        if crawl is not None and (crawl.retry_at is None or crawl.retry_at <= now):
+            # extra_filter pins the read-time retry_at so a concurrent worker
+            # that grabbed the lease between our SELECT and UPDATE wins.
+            crawl.safe_update(
+                {"retry_at": now},
+                refresh=False,
+                extra_filter={"retry_at": crawl.retry_at},
+            )
+    # Only a foreground `archivebox add` gets the interactive "abort current
+    # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
+    # orchestrators should shut down immediately and cleanly on the first signal.
+    interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
+    try:
+        with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
+            run_pending_crawls(
+                daemon=daemon,
+                crawl_id=crawl_id,
+                maintenance_only=maintenance_only,
+                interactive_interrupts=interactive_interrupts,
+            )
+        return 0
+    except (KeyboardInterrupt, asyncio.CancelledError):
         return 0
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
@@ -279,9 +298,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int:
     finally:
         current.refresh_from_db()
         if current.status != Process.StatusChoices.EXITED:
-            current.status = Process.StatusChoices.EXITED
-            current.ended_at = current.ended_at or timezone.now()
-            current.save(update_fields=["status", "ended_at", "modified_at"])
+            current.mark_exited()
 
 
 @click.command()
@@ -289,7 +306,8 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int:
 @click.option("--crawl-id", help="Run the crawl runner for a specific crawl only")
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
-def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str):
+@click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
+def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool):
     """
     Process queued work.
 
@@ -300,45 +318,70 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str):
     - --snapshot-id: Run one snapshot through its crawl only
     - --binary-id: Run one queued binary install directly on the bus
     """
-    if snapshot_id:
-        sys.exit(run_snapshot_worker(snapshot_id))
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
 
-    if binary_id:
+    if daemon and not snapshot_id and not binary_id and not crawl_id:
         try:
-            from archivebox.services.runner import run_binary
-
-            run_binary(binary_id)
-            sys.exit(0)
+            with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=False):
+                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
         except KeyboardInterrupt:
             sys.exit(0)
-        except Exception as e:
-            rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-            import traceback
 
-            traceback.print_exc()
-            sys.exit(1)
+    with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
+        if snapshot_id:
+            sys.exit(run_snapshot_worker(snapshot_id))
 
-    if crawl_id:
-        sys.exit(run_runner(daemon=False, crawl_id=crawl_id))
+        if binary_id:
+            try:
+                from archivebox.services.runner import run_binary
 
-    if daemon:
-        sys.exit(run_runner(daemon=True))
+                run_binary(binary_id)
+                sys.exit(0)
+            except KeyboardInterrupt:
+                sys.exit(0)
+            except Exception as e:
+                rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+                import traceback
 
-    if not sys.stdin.isatty():
-        sys.exit(process_stdin_records())
-    else:
-        sys.exit(run_runner(daemon=daemon))
+                traceback.print_exc()
+                sys.exit(1)
+
+        if crawl_id:
+            sys.exit(run_runner(daemon=False, crawl_id=crawl_id, maintenance_only=maintenance_only))
+
+        if maintenance_only:
+            sys.exit(run_runner(daemon=daemon, maintenance_only=True))
+
+        if not sys.stdin.isatty():
+            sys.exit(process_stdin_records())
+        else:
+            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
 
 
 def run_snapshot_worker(snapshot_id: str) -> int:
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.core.models import Snapshot
-    from archivebox.services.runner import run_crawl
+    from archivebox.services.runner import run_due_snapshot
+    from django.utils import timezone
 
+    snapshot = None
     try:
-        snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)])
+        with foreground_shutdown_signals(), foreground_parent_watchdog():
+            snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+            if snapshot.retry_at is None:
+                snapshot.update_and_requeue(retry_at=timezone.now())
+            run_due_snapshot(snapshot, lock_seconds=60)
         return 0
     except KeyboardInterrupt:
+        try:
+            if snapshot is not None:
+                snapshot.refresh_from_db()
+            else:
+                snapshot = Snapshot.objects.filter(id=snapshot_id).first()
+            if snapshot is not None and snapshot.status != Snapshot.StatusChoices.SEALED:
+                snapshot.update_and_requeue(retry_at=timezone.now())
+        except Exception:
+            pass
         return 0
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)

archivebox/cli/archivebox_schedule.py

@@ -6,7 +6,6 @@ import rich_click as click
 from rich import print
 
 from archivebox.misc.util import enforce_types, docstring
-from archivebox.config.common import get_config
 
 
 @enforce_types
@@ -20,9 +19,8 @@ def schedule(
     every: str | None = None,
     tag: str = "",
     depth: int | str = 0,
-    overwrite: bool = False,
-    update: bool | None = None,
     import_path: str | None = None,
+    config: dict[str, object] | None = None,
 ):
     """Manage database-backed scheduled crawls processed by the crawl runner."""
 
@@ -30,12 +28,10 @@ def schedule(
 
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl, CrawlSchedule
-    from archivebox.crawls.schedule_utils import validate_schedule
+    from archivebox.crawls.schedule_util import validate_schedule
     from archivebox.services.runner import run_pending_crawls
 
-    if update is None:
-        update = not get_config().ONLY_NEW
-
+    config_overrides = dict(config or {})
     depth = int(depth)
     result: dict[str, object] = {
         "created_schedule_ids": [],
@@ -79,10 +75,12 @@ def schedule(
             status=Crawl.StatusChoices.SEALED,
             retry_at=None,
             config={
-                "ONLY_NEW": not update,
-                "OVERWRITE": overwrite,
                 "DEPTH": 0 if is_update_schedule else depth,
                 "SCHEDULE_KIND": "update" if is_update_schedule else "crawl",
+                # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) win over the
+                # template defaults. Anything left unset falls through to the
+                # standard config stack at crawl-resolution time.
+                **config_overrides,
             },
         )
         crawl_schedule = CrawlSchedule.objects.create(
@@ -163,8 +161,13 @@ def schedule(
     default="0",
     help="Recursively archive linked pages up to N hops away",
 )
-@click.option("--overwrite", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
-@click.option("--update", is_flag=True, help="Retry previously failed/skipped URLs when scheduled crawls run")
+@click.option(
+    "--only-new/--no-only-new",
+    "only_new",
+    default=None,
+    help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
+    "Pass --no-only-new to force re-archive on each scheduled run.",
+)
 @click.option("--clear", is_flag=True, help="Disable all currently enabled schedules")
 @click.option("--show", is_flag=True, help="Print all currently enabled schedules")
 @click.option("--foreground", "-f", is_flag=True, help="Run the global crawl runner in the foreground (no crontab required)")
@@ -173,6 +176,9 @@ def schedule(
 @docstring(schedule.__doc__)
 def main(**kwargs):
     """Manage database-backed scheduled crawls processed by the crawl runner."""
+    only_new = kwargs.pop("only_new", None)
+    if only_new is not None:
+        kwargs["config"] = {"ONLY_NEW": bool(only_new)}
     schedule(**kwargs)
 
 

archivebox/cli/archivebox_server.py

@@ -4,120 +4,239 @@ __package__ = "archivebox.cli"
 
 import sys
 import os
+import socket
+import subprocess
+import time
 from collections.abc import Iterable
 
 import rich_click as click
 from rich import print
 
 from archivebox.misc.util import docstring, enforce_types
-from archivebox.config.common import get_config
 
 
-def stop_existing_background_runner(*, machine, process_model, supervisor=None, stop_worker_fn=None, log=print) -> int:
-    """Stop any existing orchestrator process so the server can take ownership."""
-    process_model.cleanup_stale_running(machine=machine)
-    process_model.cleanup_orphaned_workers()
+import re as _re
 
-    running_runners = list(
-        process_model.objects.filter(
-            machine=machine,
-            status=process_model.StatusChoices.RUNNING,
-            process_type=process_model.TypeChoices.ORCHESTRATOR,
-        ).order_by("created_at"),
-    )
-
-    if not running_runners:
-        return 0
-
-    log("[yellow][*] Stopping existing ArchiveBox background runner...[/yellow]")
-
-    if supervisor is not None and stop_worker_fn is not None:
-        for worker_name in ("worker_runner", "worker_runner_watch"):
-            try:
-                stop_worker_fn(supervisor, worker_name)
-            except Exception:
-                pass
-
-    for proc in running_runners:
-        try:
-            proc.kill_tree(graceful_timeout=2.0)
-        except Exception:
-            try:
-                proc.terminate(graceful_timeout=2.0)
-            except Exception:
-                pass
-
-    process_model.cleanup_stale_running(machine=machine)
-    return len(running_runners)
+_IPV4_RE = _re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
+_IPV6_CHARS_RE = _re.compile(r"^[0-9a-fA-F:.]+$")
+_LOCAL_BIND_HOSTS = frozenset({"0.0.0.0", "::", "::0", "127.0.0.1", "::1"})
 
 
-def _read_supervisor_worker_command(worker_name: str) -> str:
-    from archivebox.workers.supervisord_util import WORKERS_DIR_NAME, get_sock_file
-
-    worker_conf = get_sock_file().parent / WORKERS_DIR_NAME / f"{worker_name}.conf"
-    if not worker_conf.exists():
-        return ""
-
-    for line in worker_conf.read_text().splitlines():
-        if line.startswith("command="):
-            return line.removeprefix("command=").strip()
-    return ""
+def _is_ipv4_literal(host: str) -> bool:
+    return bool(_IPV4_RE.match(host))
 
 
-def _worker_command_matches_bind(command: str, host: str, port: str) -> bool:
-    if not command:
+def _is_ipv6_literal(host: str) -> bool:
+    # Bracketed (e.g. ``[2001:db8::1]``) or bare form. Require at least two
+    # colons so we don't catch random strings with one ``:``.
+    stripped = host.strip("[]")
+    return stripped.count(":") >= 2 and bool(_IPV6_CHARS_RE.match(stripped))
+
+
+def _bind_host_looks_like_ip(host: str) -> bool:
+    if not host or host in _LOCAL_BIND_HOSTS:
         return False
-    return f"{host}:{port}" in command or (f"--bind={host}" in command and f"--port={port}" in command)
+    return _is_ipv4_literal(host) or _is_ipv6_literal(host)
 
 
-def stop_existing_server_workers(*, supervisor, stop_worker_fn, host: str, port: str, log=print) -> int:
-    """Stop existing ArchiveBox web workers if they already own the requested bind."""
-    stopped = 0
+def _split_bind_spec(spec: str) -> tuple[str, str]:
+    """Split a ``host:port`` / ``host`` / ``port`` spec into ``(host, port)``.
+
+    The empty strings stand in for "not provided"; the caller fills in
+    defaults. Bracketed IPv6 literals like ``[::1]:8000`` are handled.
+    """
+    spec = (spec or "").strip()
+    if not spec:
+        return "", ""
+    if spec.startswith("["):
+        # Bracketed IPv6: ``[::1]`` or ``[::1]:8000``
+        end = spec.find("]")
+        if end == -1:
+            return spec, ""  # malformed; let validator reject it
+        host = spec[: end + 1]
+        rest = spec[end + 1 :]
+        if rest.startswith(":"):
+            return host, rest[1:]
+        return host, ""
+    if ":" in spec:
+        host, _, port = spec.rpartition(":")
+        return host, port
+    # Bare token: digits = port, anything else = host
+    if spec.isdigit():
+        return "", spec
+    return spec, ""
+
+
+def _parse_and_validate_bind_spec(spec: str) -> tuple[str, str]:
+    """Resolve a CLI/config bind spec to ``(host, port)`` or hard-error.
+
+    Accepts only IP literals (v4 or v6) or the special string ``localhost``
+    (normalized to ``127.0.0.1``). Bare hostnames are rejected because the
+    bind address feeds Daphne, which has to listen on a numeric address;
+    public hostnames belong in ``BASE_URL`` instead. Empty values fall back
+    to ``127.0.0.1`` / ``8000``.
+    """
+    raw_host, raw_port = _split_bind_spec(spec)
+    host = raw_host.strip()
+    port = (raw_port or "").strip() or "8000"
+
+    if host == "" or host.lower() == "localhost":
+        host = "127.0.0.1"
+    elif _is_ipv4_literal(host) or _is_ipv6_literal(host):
+        pass
+    else:
+        print(
+            f"[red][X] Invalid BIND_ADDR host {host!r}: must be an IP literal or 'localhost'.[/red]",
+        )
+        print(
+            "[red]    Hostnames like archive.example.com are not valid bind addresses — Daphne[/red]",
+        )
+        print(
+            "[red]    listens on numeric addresses only. Bind to 0.0.0.0 and set BASE_URL instead:[/red]",
+        )
+        print(
+            f"[red]      BASE_URL=https://{host} archivebox server 0.0.0.0:{port}[/red]",
+        )
+        sys.exit(1)
+
+    try:
+        port_int = int(port)
+    except ValueError:
+        print(f"[red][X] Invalid BIND_ADDR port {port!r}: must be an integer 1-65535.[/red]")
+        sys.exit(1)
+    if not (0 < port_int < 65536):
+        print(f"[red][X] Invalid BIND_ADDR port {port_int}: must be 1-65535.[/red]")
+        sys.exit(1)
+
+    return host, port
+
+
+def _print_server_startup_warnings(config, host: str, port: str) -> None:
+    """Print startup-time security / routing warnings for the server command.
+
+    Runs only from ``archivebox server`` so other entry points (manage shell,
+    plugin lookups, etc.) don't repeat this banner on every config load.
+    """
+    if config.IS_LOWER_SECURITY_MODE:
+        print(
+            f"[yellow][!] WARNING: ArchiveBox is running with SERVER_SECURITY_MODE={config.SERVER_SECURITY_MODE}[/yellow]",
+        )
+        print("[yellow]    Archived pages may share an origin with privileged app routes in this mode.[/yellow]")
+        print("[yellow]    To switch to the safer isolated setup:[/yellow]")
+        print("[yellow]      1. Set SERVER_SECURITY_MODE=safe-subdomains-fullreplay[/yellow]")
+        print("[yellow]      2. Point *.archivebox.localhost (or your chosen base domain) at this server[/yellow]")
+        print(
+            "[yellow]      3. Configure wildcard DNS/TLS or your reverse proxy so admin., web., api., and snapshot subdomains resolve[/yellow]",
+        )
+        print()
+
+    # ``config.BASE_URL`` is the merged value (env > Machine.config > file >
+    # default), which is what the running server will actually use. Earlier we
+    # gated the "BASE_URL not set" warning on ``os.environ["BASE_URL"]`` alone,
+    # which fired noisily when the user pinned BASE_URL via Machine.config /
+    # ArchiveBox.conf instead of via env.
+    base_url = (config.BASE_URL or "").strip()
+    if base_url:
+        # BASE_URL is pinned. The only thing left to surface is a port
+        # mismatch — bind port ≠ BASE_URL's explicit port usually means the
+        # operator started the server with the wrong ``archivebox server PORT``
+        # argument (or forgot to update one side after moving the listener).
+        # A reverse-proxy setup typically omits the port in BASE_URL
+        # (``https://archive.example.com``), so we only warn when BASE_URL
+        # carries an explicit port — otherwise we'd nag every proxy deployment.
+        from urllib.parse import urlparse
 
-    for worker_name in ("worker_runserver", "worker_daphne"):
         try:
-            proc = supervisor.getProcessInfo(worker_name) if supervisor else None
-        except Exception:
-            proc = None
-        if not isinstance(proc, dict) or proc.get("statename") != "RUNNING":
-            continue
+            base_port = urlparse(base_url).port
+        except (ValueError, TypeError):
+            base_port = None
+        if base_port is not None and str(base_port) != str(port):
+            print(
+                f"[yellow][!] BASE_URL ({base_url}) port {base_port} does not match the port the server is running on ({port}). "
+                "Make sure this is intentional![/yellow]",
+            )
+            print()
+        return
 
-        command = _read_supervisor_worker_command(worker_name)
-        if not _worker_command_matches_bind(command, host, port):
-            continue
+    # If the user is upgrading from 0.7.3 and already had
+    # CSRF_TRUSTED_ORIGINS set, get_base_url() will silently use that as the
+    # implicit BASE_URL. Surface what we picked so the user knows where their
+    # links / redirects are going — and tell them how to make it explicit.
+    from archivebox.core.routes_util import derive_base_url_from_csrf
 
-        if stopped == 0:
-            log("[yellow][*] Taking over existing ArchiveBox web server on same port...[/yellow]")
-        stop_worker_fn(supervisor, worker_name)
-        stopped += 1
+    csrf_derived = derive_base_url_from_csrf(config)
+    if csrf_derived:
+        print(
+            f"[yellow][!] BASE_URL is not set; auto-derived [bold]{csrf_derived}[/bold] from a single CSRF_TRUSTED_ORIGINS entry.[/yellow]",
+        )
+        print(
+            "[yellow]    Links / redirects / cookies will use that origin. To silence this hint, set BASE_URL[/yellow]",
+        )
+        print(
+            f"[yellow]    explicitly: [bold]BASE_URL={csrf_derived}[/bold] (matches your existing CSRF_TRUSTED_ORIGINS).[/yellow]",
+        )
+        print()
+        return
 
-    return stopped
+    # BASE_URL was not set explicitly. The routes_util derivation gives one of
+    # three results, with very different risk profiles — show a tailored hint
+    # so new users coming from the 0.7.x single-domain world know whether the
+    # default is fine for them or needs attention.
+    if _bind_host_looks_like_ip(host):
+        # Real IP literal: subdomain routing can't work, URLs leak the IP.
+        # This is the most urgent case.
+        print(
+            f"[yellow][!] WARNING: BASE_URL is not set and BIND_ADDR resolves to an IP literal ({host}).[/yellow]",
+        )
+        print(
+            "[yellow]    Snapshot / admin / api URLs will be generated with the IP, and subdomain[/yellow]",
+        )
+        print(
+            "[yellow]    routing cannot work against an IP address. Set BASE_URL explicitly, e.g.[/yellow]",
+        )
+        print(
+            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
+        )
+        if config.USES_SUBDOMAIN_ROUTING:
+            print(
+                "[yellow]    Or switch SERVER_SECURITY_MODE to a one-domain mode if you can't run a hostname.[/yellow]",
+            )
+        print()
+    else:
+        # Loopback / wildcard bind. The routes_util default of
+        # http://archivebox.localhost:PORT works in a browser on the same
+        # machine, but anything else (reverse proxy, k8s ingress, LAN client)
+        # needs BASE_URL set. (Real hostnames can't reach this branch — the
+        # bind validator rejects them upfront.)
+        print(
+            "[yellow][!] BASE_URL is not set. Generated URLs will fall back to http://archivebox.localhost:<port>.[/yellow]",
+        )
+        print(
+            "[yellow]    That's fine for local browsing on this machine. Set BASE_URL when running behind[/yellow]",
+        )
+        print(
+            "[yellow]    a reverse proxy / ingress / public hostname, e.g.[/yellow]",
+        )
+        print(
+            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
+        )
+        print()
 
 
 @enforce_types
 def server(
     runserver_args: Iterable[str] | None = None,
     reload: bool = False,
-    init: bool = False,
     debug: bool = False,
     daemonize: bool = False,
     nothreading: bool = False,
 ) -> None:
     """Run the ArchiveBox HTTP server"""
+    from archivebox.config.common import get_config
 
     config = get_config()
     runserver_args = list(runserver_args or (config.BIND_ADDR,))
 
-    if init:
-        from archivebox.cli.archivebox_init import init as archivebox_init
-
-        archivebox_init(quick=True)
-        print()
-
-    from archivebox.misc.checks import check_data_folder
-
-    check_data_folder()
-
     run_in_debug = config.DEBUG or debug or reload
     if debug or reload:
         os.environ["DEBUG"] = "True"
@@ -132,90 +251,142 @@ def server(
         print("      [green]archivebox manage createsuperuser[/green]")
         print()
 
-    host = "127.0.0.1"
-    port = "8000"
+    # First non-empty positional arg is the bind spec; otherwise inherit from
+    # config (which defaults to "127.0.0.1:8000"). _parse_and_validate_bind_spec
+    # hard-errors on hostnames so the rest of the server can assume a numeric
+    # bind host.
+    bind_spec = next((arg for arg in runserver_args if arg), "")
+    host, port = _parse_and_validate_bind_spec(bind_spec)
 
-    try:
-        host_and_port = [arg for arg in runserver_args if arg.replace(".", "").replace(":", "").isdigit()][0]
-        if ":" in host_and_port:
-            host, port = host_and_port.split(":")
+    if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1":
+        from archivebox.config import CONSTANTS
+
+        log_path = CONSTANTS.LOGS_DIR / "server.log"
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        daemon_env = os.environ.copy()
+        daemon_env["ARCHIVEBOX_SERVER_DAEMON_CHILD"] = "1"
+        daemon_cmd = [sys.executable, "-m", "archivebox", "server"]
+        if debug:
+            daemon_cmd.append("--debug")
+        if reload:
+            daemon_cmd.append("--reload")
+        if nothreading:
+            daemon_cmd.append("--nothreading")
+        daemon_cmd.extend(runserver_args)
+        with log_path.open("a", encoding="utf-8") as log_file:
+            proc = subprocess.Popen(
+                daemon_cmd,
+                cwd=os.getcwd(),
+                env=daemon_env,
+                stdin=subprocess.DEVNULL,
+                stdout=log_file,
+                stderr=log_file,
+                start_new_session=True,
+            )
+        deadline = time.monotonic() + 30
+        while time.monotonic() < deadline:
+            if proc.poll() is not None:
+                print(f"[red][X] ArchiveBox daemon server exited early with code {proc.returncode}. See {log_path}[/red]")
+                sys.exit(proc.returncode or 1)
+            try:
+                with socket.create_connection((host, int(port)), timeout=0.25):
+                    break
+            except OSError:
+                time.sleep(0.1)
         else:
-            if "." in host_and_port:
-                host = host_and_port
-            else:
-                port = host_and_port
-    except IndexError:
-        pass
+            print(f"[yellow][!] ArchiveBox daemon server pid={proc.pid} is still starting. See {log_path}[/yellow]")
+        return
+
+    os.environ["BIND_ADDR"] = f"{host}:{port}"
+    from archivebox.core.routes_util import get_base_url
+
+    base_url = get_base_url().rstrip("/")
+    admin_url = f"{base_url}/admin/"
 
     from archivebox.workers.supervisord_util import (
-        get_existing_supervisord_process,
-        get_worker,
-        stop_worker,
+        active_supervisord_runtime_components,
+        format_runtime_components,
         start_server_workers,
+        stop_existing_supervisord_process,
         is_port_in_use,
     )
-    from archivebox.machine.models import Machine, Process
-
-    machine = Machine.current()
-    supervisor = get_existing_supervisord_process()
-    stop_existing_background_runner(
-        machine=machine,
-        process_model=Process,
-        supervisor=supervisor,
-        stop_worker_fn=stop_worker,
+    from archivebox.machine.models import Process
+    from archivebox.core.takeover_util import (
+        command_owns_runtime_stack,
+        current_command,
+        runtime_stack_owner,
+        standby_until_runtime_stack_needed,
     )
-    if supervisor:
-        stop_existing_server_workers(
-            supervisor=supervisor,
-            stop_worker_fn=stop_worker,
-            host=host,
-            port=port,
-        )
-
-    # Check if port is already in use
-    if is_port_in_use(host, int(port)):
-        print(f"[red][X] Error: Port {port} is already in use[/red]")
-        print(f"    Another process (possibly daphne or runserver) is already listening on {host}:{port}")
-        print("    Stop the conflicting process or choose a different port")
-        sys.exit(1)
-
-    supervisor = get_existing_supervisord_process()
-    if supervisor:
-        server_worker_name = "worker_runserver" if run_in_debug else "worker_daphne"
-        server_proc = get_worker(supervisor, server_worker_name)
-        server_state = server_proc.get("statename") if isinstance(server_proc, dict) else None
-        if server_state == "RUNNING":
-            runner_proc = get_worker(supervisor, "worker_runner")
-            runner_watch_proc = get_worker(supervisor, "worker_runner_watch")
-            runner_state = runner_proc.get("statename") if isinstance(runner_proc, dict) else None
-            runner_watch_state = runner_watch_proc.get("statename") if isinstance(runner_watch_proc, dict) else None
-            print("[red][X] Error: ArchiveBox server is already running[/red]")
-            print(
-                f"    [green]√[/green] Web server ({server_worker_name}) is RUNNING on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
-            )
-            if runner_state == "RUNNING":
-                print("    [green]√[/green] Background runner (worker_runner) is RUNNING")
-            if runner_watch_state == "RUNNING":
-                print("    [green]√[/green] Reload watcher (worker_runner_watch) is RUNNING")
-            print()
-            print("[yellow]To stop the existing server, run:[/yellow]")
-            print('    pkill -f "archivebox server"')
-            print("    pkill -f supervisord")
-            sys.exit(1)
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
 
     if run_in_debug:
         print("[green][+] Starting ArchiveBox webserver in DEBUG mode...[/green]")
     else:
         print("[green][+] Starting ArchiveBox webserver...[/green]")
     print(
-        f"    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
+        f"    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [dim]BIND_ADDR[/dim] [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
     )
     print(
-        f"    [green]>[/green] Log in to ArchiveBox Admin UI on [deep_sky_blue3][link=http://{host}:{port}/admin]http://{host}:{port}/admin[/link][/deep_sky_blue3]",
+        f"    [green]>[/green] Log in to ArchiveBox Admin UI on [dim]BASE_URL [/dim] [deep_sky_blue3][link={admin_url}]{admin_url}[/link][/deep_sky_blue3]",
     )
     print("    > Writing ArchiveBox error log to ./logs/errors.log")
     print()
-    start_server_workers(host=host, port=port, daemonize=daemonize, debug=run_in_debug, reload=reload, nothreading=nothreading)
+
+    # Reload config after we've set os.environ["BIND_ADDR"] above so the
+    # security-mode + base-url warnings see the effective values.
+    runtime_config = get_config()
+    _print_server_startup_warnings(runtime_config, host, port)
+    bind_url = f"http://{host}:{port}"
+    command = current_command(Process.TypeChoices.SERVER, data_dir=config.DATA_DIR, url=bind_url)
+
+    def still_owns_runtime_stack() -> bool:
+        from django.db import connections
+
+        try:
+            return command_owns_runtime_stack(command, data_dir=config.DATA_DIR)
+        finally:
+            connections.close_all()
+
+    try:
+        with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"):
+            while True:
+                standby_result = standby_until_runtime_stack_needed(command, data_dir=config.DATA_DIR)
+                older_owner = runtime_stack_owner(data_dir=config.DATA_DIR, exclude_id=command.id)
+                takeover_components = active_supervisord_runtime_components(config=config)
+                if older_owner and takeover_components:
+                    print(
+                        "[yellow][*] Taking over "
+                        f"{format_runtime_components(takeover_components)} from older existing archivebox process (pid={older_owner.pid}).[/yellow]",
+                    )
+                stop_existing_supervisord_process()
+                if is_port_in_use(host, int(port)):
+                    print(f"[red][X] Error: Port {port} is already in use[/red]")
+                    print(f"    Another process outside this ArchiveBox runtime is listening on {host}:{port}")
+                    sys.exit(1)
+
+                result = start_server_workers(
+                    host=host,
+                    port=port,
+                    daemonize=False,
+                    debug=run_in_debug,
+                    reload=reload,
+                    nothreading=nothreading,
+                    keep_running=still_owns_runtime_stack,
+                    should_stop_supervisord=still_owns_runtime_stack,
+                    resumed_from_pid=standby_result.get("previous_owner_pid") if standby_result.get("resumed") else None,
+                )
+                if result == "interrupted":
+                    break
+                if not still_owns_runtime_stack():
+                    continue
+                if result == "exited":
+                    print("[yellow][*] Runtime stack exited while this parent is still leader; restarting...[/yellow]")
+                    continue
+                break
+    except KeyboardInterrupt:
+        pass
+    finally:
+        command.mark_exited()
     print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")
 
 
@@ -224,7 +395,6 @@ def server(
 @click.option("--reload", is_flag=True, help="Enable auto-reloading when code or templates change")
 @click.option("--debug", is_flag=True, help="Enable DEBUG=True mode with more verbose errors")
 @click.option("--nothreading", is_flag=True, help="Force runserver to run in single-threaded mode")
-@click.option("--init", is_flag=True, help="Run a full archivebox init/upgrade before starting the server")
 @click.option("--daemonize", is_flag=True, help="Run the server in the background as a daemon")
 @docstring(server.__doc__)
 def main(**kwargs):

archivebox/cli/archivebox_snapshot.py

@@ -35,10 +35,9 @@ from collections.abc import Iterable
 
 import rich_click as click
 from rich import print as rprint
-from django.db.models import Case, IntegerField, Q, Sum, When
-from django.db.models.functions import Coalesce
+from django.db.models import Case, IntegerField, QuerySet, When
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================
@@ -179,6 +178,60 @@ def create_snapshots(
 # =============================================================================
 
 
+def build_snapshot_queryset(
+    *,
+    status: str | None = None,
+    url__icontains: str | None = None,
+    url__istartswith: str | None = None,
+    tag: str | None = None,
+    crawl_id: str | None = None,
+    sort: str | None = None,
+    search: str | None = None,
+    query: str | None = None,
+    limit: int | None = None,
+) -> QuerySet:
+    from archivebox.core.models import Snapshot
+    from archivebox.search.query import apply_snapshot_search
+
+    queryset = Snapshot.objects.order_by("-created_at")
+    queryset = apply_filters(
+        queryset,
+        {
+            "status": status,
+            "url__icontains": url__icontains,
+            "url__istartswith": url__istartswith,
+            "crawl_id": crawl_id,
+        },
+    )
+
+    if tag:
+        queryset = queryset.filter(tags__name__iexact=tag)
+
+    query = (query or "").strip()
+    if query:
+        try:
+            queryset = apply_snapshot_search(
+                queryset,
+                query,
+                search_mode=search,
+                ordering=("-created_at",) if not sort else None,
+                max_results=limit,
+                skip_backend_when_metadata_satisfies_limit=True,
+                include_metadata_for_forced_backend=True,
+            )
+        except Exception as err:
+            rprint(
+                f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
+                file=sys.stderr,
+            )
+            queryset = apply_snapshot_search(queryset, query, search_mode="meta")
+
+    if sort:
+        queryset = queryset.order_by(sort)
+
+    return queryset
+
+
 def list_snapshots(
     status: str | None = None,
     url__icontains: str | None = None,
@@ -200,12 +253,6 @@ def list_snapshots(
     """
     from archivebox.misc.jsonl import write_record
     from archivebox.core.models import Snapshot
-    from archivebox.search import (
-        get_default_search_mode,
-        get_search_mode,
-        prioritize_metadata_matches,
-        query_search_index,
-    )
 
     if with_headers and not csv:
         rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr)
@@ -213,57 +260,17 @@ def list_snapshots(
 
     is_tty = sys.stdout.isatty() and not csv
 
-    queryset = Snapshot.objects.order_by("-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "url__icontains": url__icontains,
-        "url__istartswith": url__istartswith,
-        "crawl_id": crawl_id,
-    }
-    queryset = apply_filters(queryset, filter_kwargs)
-
-    # Tag filter requires special handling (M2M)
-    if tag:
-        queryset = queryset.filter(tags__name__iexact=tag)
-
-    query = (query or "").strip()
-    if query:
-        metadata_qs = queryset.filter(
-            Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query),
-        )
-        requested_search_mode = (search or "").strip().lower()
-        if requested_search_mode == "content":
-            requested_search_mode = "contents"
-        search_mode = get_default_search_mode() if not requested_search_mode else get_search_mode(requested_search_mode)
-
-        if search_mode == "meta":
-            queryset = metadata_qs
-        else:
-            try:
-                deep_qsearch = None
-                if search_mode == "deep":
-                    qsearch = query_search_index(query, search_mode="contents")
-                    deep_qsearch = query_search_index(query, search_mode="deep")
-                else:
-                    qsearch = query_search_index(query, search_mode=search_mode)
-                queryset = prioritize_metadata_matches(
-                    queryset,
-                    metadata_qs,
-                    qsearch,
-                    deep_queryset=deep_qsearch,
-                    ordering=("-created_at",) if not sort else None,
-                )
-            except Exception as err:
-                rprint(
-                    f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
-                    file=sys.stderr,
-                )
-                queryset = metadata_qs
-
-    if sort:
-        queryset = queryset.order_by(sort)
+    queryset = build_snapshot_queryset(
+        status=status,
+        url__icontains=url__icontains,
+        url__istartswith=url__istartswith,
+        tag=tag,
+        crawl_id=crawl_id,
+        sort=sort,
+        search=search,
+        query=query,
+        limit=limit,
+    )
 
     if not is_tty:
         if limit:
@@ -273,7 +280,7 @@ def list_snapshots(
                 output_field=IntegerField(),
             )
             queryset = Snapshot.objects.filter(id__in=limited_ids).order_by(preserved_order)
-        queryset = queryset.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)).prefetch_related("tags")
+        queryset = queryset.prefetch_related("tags")
     elif limit:
         queryset = queryset[:limit]
 
@@ -381,22 +388,34 @@ def update_snapshots(
         try:
             snapshot = Snapshot.objects.get(id=snapshot_id)
 
-            # Apply updates from CLI flags (override stdin values)
             if status:
-                snapshot.status = status
-                snapshot.retry_at = timezone.now()
+                if status not in Snapshot.StatusChoices.values:
+                    rprint(f"[red]Invalid snapshot status: {status}[/red]", file=sys.stderr)
+                    continue
+                if status == Snapshot.StatusChoices.SEALED:
+                    snapshot.cancel()
+                elif status == Snapshot.StatusChoices.PAUSED:
+                    snapshot.pause()
+                elif status == Snapshot.StatusChoices.QUEUED:
+                    if snapshot.status == Snapshot.StatusChoices.PAUSED:
+                        snapshot.resume()
+                    else:
+                        snapshot.update_and_requeue(status=Snapshot.StatusChoices.QUEUED, retry_at=timezone.now())
+                elif status == Snapshot.StatusChoices.STARTED:
+                    snapshot.update_and_requeue(status=Snapshot.StatusChoices.STARTED, retry_at=timezone.now())
             if tag:
-                # Add tag to existing tags
-                snapshot.save()  # Ensure saved before M2M
                 from archivebox.core.models import Tag
 
                 tag_obj, _ = Tag.objects.get_or_create(name=tag)
                 snapshot.tags.add(tag_obj)
+                snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
 
-            snapshot.save()
+            if not status and not tag:
+                snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
             updated_count += 1
 
             if not is_tty:
+                snapshot.refresh_from_db()
                 write_record(snapshot.to_json())
 
         except Snapshot.DoesNotExist:
@@ -490,7 +509,7 @@ def create_cmd(urls: tuple, tag: str, status: str, depth: int):
 @click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
 @click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
 @click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
+@click.option("--search", help="Search mode to use for the query")
 @click.argument("query", nargs=-1)
 def list_cmd(
     status: str | None,

archivebox/cli/archivebox_status.py

@@ -61,7 +61,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
             num_dirs += root_dirs
             num_files += root_files
     else:
-        num_bytes = ArchiveResult.objects.aggregate(total=Coalesce(Sum("output_size"), 0))["total"] or 0
+        num_bytes = snapshots_qs.aggregate(total=Coalesce(Sum("output_size"), 0))["total"] or 0
         num_dirs = 0
         num_files = ArchiveResult.objects.exclude(output_files__in=["", "{}"]).count()
     size = printable_filesize(num_bytes)
@@ -130,7 +130,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
         print("        [green]archivebox manage createsuperuser[/green]")
 
     print()
-    recent_snapshots = snapshots_qs.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)).order_by(
+    recent_snapshots = snapshots_qs.order_by(
         "-downloaded_at",
         "-modified_at",
     )[:10]
@@ -141,7 +141,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
             (
                 "[grey53] "
                 f"   > {str(snapshot.downloaded_at)[:16]} "
-                f"[{snapshot.num_outputs} {('X', '√')[snapshot.status == Snapshot.StatusChoices.SEALED]} {printable_filesize(snapshot.output_size_sum or 0)}] "
+                f"[{snapshot.num_outputs} {('X', '√')[snapshot.status == Snapshot.StatusChoices.SEALED]} {printable_filesize(snapshot.output_size or 0)}] "
                 f'"{snapshot.title}": {snapshot.url}'
                 "[/grey53]"
             )[: config.TERM_WIDTH],

archivebox/cli/archivebox_tag.py

@@ -36,7 +36,7 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================

archivebox/config/__init__.py

@@ -3,9 +3,21 @@
 __package__ = "archivebox.config"
 __order__ = 200
 
-from .paths import (
-    PACKAGE_DIR,
-    DATA_DIR,
-)
-from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR  # noqa
-from .version import VERSION  # noqa
+
+def __getattr__(name: str):
+    if name in ("CONSTANTS", "CONSTANTS_CONFIG"):
+        from .constants import CONSTANTS, CONSTANTS_CONFIG
+
+        return {"CONSTANTS": CONSTANTS, "CONSTANTS_CONFIG": CONSTANTS_CONFIG}[name]
+    if name in ("PACKAGE_DIR", "DATA_DIR"):
+        from .paths import PACKAGE_DIR, DATA_DIR
+
+        return {"PACKAGE_DIR": PACKAGE_DIR, "DATA_DIR": DATA_DIR}[name]
+    if name == "VERSION":
+        from .version import VERSION
+
+        return VERSION
+    raise AttributeError(name)
+
+
+__all__ = ("CONSTANTS", "CONSTANTS_CONFIG", "PACKAGE_DIR", "DATA_DIR", "VERSION")

archivebox/config/collection.py

@@ -1,37 +1,296 @@
 __package__ = "archivebox.config"
 
+import io
+import json
 import os
-
-from benedict import benedict
+from typing import Any
 
 from archivebox.config.constants import CONSTANTS
 from archivebox.config.configset import CaseConfigParser
+from archivebox.misc.logging import AttrDict
 
 
-def write_config_file(config: dict[str, str]) -> benedict:
-    """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
+CONFIG_FILE_HEADER = (
+    "# This is the config file for your ArchiveBox collection.\n"
+    "#\n"
+    "# You can add options here manually in INI format, or automatically by running:\n"
+    "#    archivebox config --set KEY=VALUE\n"
+    "#\n"
+    "# This file is kept in sync 1:1 with Machine.config in the index DB —\n"
+    "# editing either side propagates to the other. ``archivebox init`` reads\n"
+    "# this file on startup; the admin Machine.config editor writes both.\n"
+    "#\n"
+    "# Full reference:\n"
+    "#    https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration\n"
+    "\n"
+)
 
+
+# Recursion guard for the bidirectional file<->DB mirror. Bidirectional sync
+# means a write on one side always triggers an update on the other; without
+# this flag, ``Machine.save -> mirror to file -> write_config_file -> mirror
+# back to Machine -> Machine.save -> ...`` would loop forever. Module-level
+# scalar is fine here — daphne handles each request on its own thread and the
+# admin/CLI write paths are inherently serialized through the DB.
+_MIRROR_IN_PROGRESS: bool = False
+# One-time-per-process startup sync, gated so subsequent ``Machine.current()``
+# calls collapse to a single boolean check.
+_INITIAL_SYNC_DONE: bool = False
+
+
+def _coerce_to_str_dict(config: Any) -> dict[str, str]:
+    """Project an arbitrary config payload to flat ``{UPPER_KEY: str}`` form.
+
+    INI files only round-trip strings, so we normalize Machine.config values
+    to strings on the way out and accept the same shape on the way back.
+    Pydantic re-coerces types at read time inside ``get_config``.
+
+    Composite values (``dict`` / ``list`` / ``tuple``) are JSON-encoded so
+    they round-trip back through pydantic-settings — ``str(some_dict)``
+    would produce Python's repr (``{'k': 'v'}`` with single quotes), and
+    pydantic-settings refuses to parse that as a ``dict`` field, causing
+    e.g. ``ABX_INSTALL_CACHE`` to crash with ``ValidationError: Input
+    should be a valid dictionary``.
+    """
+    if not config:
+        return {}
+    if not hasattr(config, "items"):
+        return {}
+    flat: dict[str, str] = {}
+    for key, value in config.items():
+        upper_key = str(key).upper()
+        if value is None:
+            flat[upper_key] = ""
+        elif isinstance(value, (dict, list, tuple)):
+            flat[upper_key] = json.dumps(value, default=str)
+        else:
+            flat[upper_key] = str(value)
+    return flat
+
+
+def _load_file_config_dict() -> tuple[dict[str, str], float | None]:
+    """Return ``(flat_dict, mtime)`` for ``ArchiveBox.conf`` (``({}, None)`` if missing)."""
+    config_path = CONSTANTS.CONFIG_FILE
+    try:
+        mtime = config_path.stat().st_mtime
+    except FileNotFoundError:
+        return {}, None
+    parser = CaseConfigParser()
+    parser.read(config_path)
+    flat = {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+    return flat, mtime
+
+
+def _resolve_section_for_key(key: str, config_sections, plugin_configs) -> str:
+    for section in config_sections.values():
+        if key in type(section).model_fields:
+            return section.toml_section_header
+    for schema in plugin_configs.values():
+        if "properties" in schema and key in schema["properties"]:
+            return "PLUGINS"
+    # Unknown / user-defined keys land in SERVER_CONFIG so we never lose them
+    # (the previous code raised here, which was fine for the CLI-only path but
+    # would break the mirror as soon as anyone added a plugin-tunable that
+    # this process hasn't loaded a schema for).
+    return "SERVER_CONFIG"
+
+
+def _render_config_file_content(config: dict[str, str]) -> str:
+    """Render a flat config dict to INI text, grouped by inferred section."""
     from archivebox.config.common import get_all_configs
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
+
+    config_sections = get_all_configs()
+    plugin_configs = discover_plugin_configs()
+
+    parser = CaseConfigParser()
+    for key, val in sorted(config.items()):
+        section = _resolve_section_for_key(key, config_sections, plugin_configs)
+        if section not in parser:
+            parser[section] = {}
+        parser[section][key] = "" if val is None else str(val)
+
+    buf = io.StringIO()
+    buf.write(CONFIG_FILE_HEADER)
+    parser.write(buf)
+    return buf.getvalue()
+
+
+def _write_file_if_changed(content: str) -> bool:
+    """Atomic-write ``ArchiveBox.conf`` only when contents actually differ.
+
+    Skipping unchanged writes is the difference between every Machine.save in
+    a hot autodetection loop costing one disk write vs. zero.
+    """
     from archivebox.misc.system import atomic_write
 
-    CONFIG_HEADER = """# This is the config file for your ArchiveBox collection.
-    #
-    # You can add options here manually in INI format, or automatically by running:
-    #    archivebox config --set KEY=VALUE
-    #
-    # If you modify this file manually, make sure to update your archive after by running:
-    #    archivebox init
-    #
-    # A list of all possible config with documentation and examples can be found here:
-    #    https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration
+    config_path = CONSTANTS.CONFIG_FILE
+    try:
+        existing = config_path.read_text(encoding="utf-8") if config_path.exists() else None
+    except OSError:
+        existing = None
+    if existing == content:
+        return False
+    atomic_write(config_path, content)
+    return True
 
+
+def mirror_machine_config_to_file(config: Any) -> None:
+    """Rewrite ``ArchiveBox.conf`` so it mirrors ``Machine.config`` exactly.
+
+    Called from ``Machine.save`` after the row is committed. Recursion-guarded
+    so the matching write_config_file -> Machine.save bounce doesn't loop.
     """
+    global _MIRROR_IN_PROGRESS
+    if _MIRROR_IN_PROGRESS:
+        return
+    _MIRROR_IN_PROGRESS = True
+    try:
+        flat = _coerce_to_str_dict(config)
+        _write_file_if_changed(_render_config_file_content(flat))
+    finally:
+        _MIRROR_IN_PROGRESS = False
+
+
+def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
+    """Inverse of ``_coerce_to_str_dict``: decode complex INI values to native.
+
+    ``mirror_machine_config_to_file`` JSON-encodes ``dict`` / ``list`` values
+    so they round-trip through INI's string-only storage. When reading the
+    file back into ``Machine.config`` (a JSONField that holds native types)
+    those strings have to be decoded — otherwise downstream consumers like
+    ``_emit_machine_config`` → ``MachineEvent`` → abx-dl see a JSON string
+    where they expect a dict and raise ``TypeError``.
+    Declared fields go through pydantic-settings' own ``field_is_complex`` /
+    ``prepare_field_value`` so they're decoded per annotation. Undeclared
+    keys (e.g. ``ABX_INSTALL_CACHE``, written dynamically by abx-dl) are
+    JSON-decoded when their string starts with ``{`` or ``[`` — the same
+    shape ``_coerce_to_str_dict`` writes them as.
+    """
+    from archivebox.config.common import ArchiveBoxConfig
+    from archivebox.config.configset import IniConfigSettingsSource
+
+    decoder = IniConfigSettingsSource(ArchiveBoxConfig)
+    decoded: dict[str, Any] = dict(file_config)
+    declared_fields = set(ArchiveBoxConfig.model_fields)
+    for field_name, field in ArchiveBoxConfig.model_fields.items():
+        if field_name not in decoded:
+            continue
+        raw = decoded[field_name]
+        if not isinstance(raw, str) or not raw:
+            continue
+        if decoder.field_is_complex(field):
+            decoded[field_name] = decoder.prepare_field_value(field_name, field, raw, True)
+    for key, raw in list(decoded.items()):
+        if key in declared_fields:
+            continue
+        if not isinstance(raw, str) or not raw:
+            continue
+        first = raw[:1]
+        if first not in ("{", "["):
+            continue
+        try:
+            decoded[key] = json.loads(raw)
+        except (TypeError, ValueError):
+            continue
+    return decoded
+
+
+def _mirror_file_to_machine_config(file_config: dict[str, str]) -> None:
+    """Copy ``ArchiveBox.conf`` contents into ``Machine.config``.
+
+    Internal helper used by ``write_config_file`` and the startup sync —
+    callers must hold the ``_MIRROR_IN_PROGRESS`` guard around it.
+    """
+    from archivebox.machine.models import Machine
+
+    machine = Machine.current()
+    if _coerce_to_str_dict(machine.config) == file_config:
+        return
+    machine.config = _coerce_from_str_dict(file_config)
+    machine.save(update_fields=["config", "modified_at"])
+
+
+def sync_machine_and_file(machine: Any = None) -> None:
+    """One-time-per-process reconciliation between the two stores.
+
+    Cheap on the common case where they already agree (single ``stat`` + dict
+    compare ≈ 1ms). When the two sides diverge we merge them: each side's
+    unique keys are preserved, and for keys present on both we let the newer
+    side win (file mtime vs. ``Machine.modified_at``). After the merge both
+    stores hold the union, so every subsequent write keeps them in lockstep
+    via the full-replace mirror functions.
+
+    Pass ``machine`` when the caller already has a current ``Machine``
+    instance (e.g. from ``Machine.current()``) to skip the 10–15ms
+    ``get_host_guid()`` round-trip on the cold path.
+    """
+    global _INITIAL_SYNC_DONE, _MIRROR_IN_PROGRESS
+    if _INITIAL_SYNC_DONE:
+        return
+    _INITIAL_SYNC_DONE = True
+    if _MIRROR_IN_PROGRESS:
+        return
+    _MIRROR_IN_PROGRESS = True
+    try:
+        if machine is None:
+            from archivebox.machine.detect import get_host_guid
+            from archivebox.machine.models import Machine
+
+            try:
+                machine = Machine.objects.filter(guid=get_host_guid()).first()
+            except Exception:
+                return
+            if machine is None:
+                return
+
+        file_config, file_mtime = _load_file_config_dict()
+        machine_config = _coerce_to_str_dict(machine.config)
+        if machine_config == file_config:
+            return
+
+        db_mtime = machine.modified_at.timestamp() if machine.modified_at else 0.0
+        file_is_newer = file_mtime is not None and file_mtime > db_mtime
+
+        merged: dict[str, str] = {}
+        all_keys = set(machine_config) | set(file_config)
+        for key in all_keys:
+            in_file = key in file_config
+            in_db = key in machine_config
+            if in_file and in_db:
+                if file_config[key] == machine_config[key]:
+                    merged[key] = file_config[key]
+                else:
+                    merged[key] = file_config[key] if file_is_newer else machine_config[key]
+            elif in_file:
+                merged[key] = file_config[key]
+            else:
+                merged[key] = machine_config[key]
+
+        if merged != file_config:
+            _write_file_if_changed(_render_config_file_content(merged))
+        if merged != machine_config:
+            machine.config = _coerce_from_str_dict(merged)
+            machine.save(update_fields=["config", "modified_at"])
+    finally:
+        _MIRROR_IN_PROGRESS = False
+
+
+def write_config_file(config: dict[str, str]) -> AttrDict:
+    """Merge ``config`` into ``ArchiveBox.conf``, validate, then mirror to Machine.config.
+
+    Backwards-compatible signature: callers (CLI ``archivebox config --set``
+    and the init flow) pass a partial dict of keys to upsert.
+    """
+
+    from archivebox.config.common import get_all_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
+    from archivebox.misc.system import atomic_write
 
     config_path = CONSTANTS.CONFIG_FILE
 
     if not os.access(config_path, os.F_OK):
-        atomic_write(config_path, CONFIG_HEADER)
+        atomic_write(config_path, CONFIG_FILE_HEADER)
 
     config_file = CaseConfigParser()
     config_file.read(config_path)
@@ -44,27 +303,13 @@ def write_config_file(config: dict[str, str]) -> benedict:
 
     # Set up sections in empty config file
     for key, val in config.items():
-        section_name = None
-        for section in config_sections.values():
-            if key in type(section).model_fields:
-                section_name = section.toml_section_header
-                break
-
-        if section_name is None:
-            for schema in plugin_configs.values():
-                if "properties" in schema and key in schema["properties"]:
-                    section_name = "PLUGINS"
-                    break
-
-        if section_name is None:
-            raise ValueError(f"No config section found for key: {key}")
-
+        section_name = _resolve_section_for_key(key, config_sections, plugin_configs)
         if section_name in config_file:
             existing_config = dict(config_file[section_name])
         else:
             existing_config = {}
 
-        config_file[section_name] = benedict({**existing_config, key: val})
+        config_file[section_name] = AttrDict({**existing_config, key: val})
 
     with open(config_path, "w+", encoding="utf-8") as new:
         config_file.write(new)
@@ -85,4 +330,18 @@ def write_config_file(config: dict[str, str]) -> benedict:
     if os.access(f"{config_path}.bak", os.F_OK):
         os.remove(f"{config_path}.bak")
 
-    return benedict({key.upper(): updated_config.get(key.upper()) for key in config.keys()})
+    # Mirror the post-write file state into Machine.config so the DB stays
+    # 1:1 with the on-disk file. Recursion-guarded so Machine.save's own
+    # mirror-back doesn't loop us.
+    global _MIRROR_IN_PROGRESS
+    if not _MIRROR_IN_PROGRESS:
+        _MIRROR_IN_PROGRESS = True
+        try:
+            flat, _mtime = _load_file_config_dict()
+            _mirror_file_to_machine_config(flat)
+        except Exception:
+            pass
+        finally:
+            _MIRROR_IN_PROGRESS = False
+
+    return AttrDict({key.upper(): updated_config.get(key.upper()) for key in config.keys()})

archivebox/config/common.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 __package__ = "archivebox.config"
 
 import json
@@ -6,16 +8,18 @@ import re
 import secrets
 import sys
 import shutil
+from functools import lru_cache
 from collections.abc import Mapping
+from datetime import timedelta
 from typing import Any, ClassVar, cast
 from pathlib import Path
 
 from rich.console import Console
-from pydantic import BaseModel, Field, create_model, field_validator, model_validator
+from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator
 from pydantic_settings import SettingsConfigDict
 from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs
 
-from archivebox.config.configset import BaseConfigSet
+from archivebox.config.configset import BaseConfigSet, IniConfigSettingsSource
 from archivebox.config.configset import COMPUTED_CONFIG_KEYS
 
 from .constants import CONSTANTS
@@ -31,10 +35,117 @@ PluginSchemaDocuments = dict[str, dict[str, Any]]
 
 _STDOUT_CONSOLE = Console()
 _STDERR_CONSOLE = Console(stderr=True)
-_WARNED_SERVER_SECURITY_MODES: set[str] = set()
 _WARNED_ARCHIVING_CONFIGS: set[tuple[int, bool]] = set()
 
 
+def _legacy_bool(value: object) -> bool | None:
+    if value is None:
+        return None
+    normalized = str(value).strip().lower()
+    if normalized in {"1", "true", "yes", "on"}:
+        return True
+    if normalized in {"0", "false", "no", "off"}:
+        return False
+    return None
+
+
+def permissions_from_legacy_public_flags(raw_config: Mapping[str, object]) -> str | None:
+    if str(raw_config.get("PERMISSIONS") or "").strip():
+        return None
+
+    public_snapshots = _legacy_bool(raw_config.get("PUBLIC_SNAPSHOTS"))
+    public_index = _legacy_bool(raw_config.get("PUBLIC_INDEX"))
+    if public_snapshots is False:
+        return "private"
+    if public_index is False:
+        return "unlisted"
+    if public_snapshots is True or public_index is True:
+        return "public"
+    return None
+
+
+_SENSITIVE_CONFIG_KEY_NEEDLES = ("TOKEN", "SECRET", "API_KEY", "APIKEY", "PASSWORD")
+SENSITIVE_CONFIG_VALUE_REDACTED = "********"
+_SCOPE_CRAWL_FROZEN = "crawl_frozen"
+_SCOPE_CRAWL_EXECUTION = "crawl_execution"
+_SCOPE_SERVER = "server"
+
+
+@lru_cache(maxsize=1)
+def _plugin_sensitive_config_keys() -> frozenset[str]:
+    sensitive_keys: set[str] = set()
+    for prop_key, prop_schema in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).items():
+        if isinstance(prop_schema, Mapping) and prop_schema.get("x-sensitive"):
+            sensitive_keys.add(str(prop_key))
+    return frozenset(sensitive_keys)
+
+
+def is_sensitive_config_key(key: str) -> bool:
+    """True if a config key names a credential and must be write-only in the UI.
+
+    Matches any key whose uppercase form contains ``TOKEN``, ``SECRET``,
+    ``API_KEY``, ``APIKEY``, or ``PASSWORD`` — covers ``SECRET_KEY``,
+    ``OPENAI_API_KEY``, ``TWOCAPTCHA_APIKEY``, ``GITHUB_TOKEN``,
+    ``ADMIN_PASSWORD``, etc. Centralized here so the KeyValueWidget
+    (Machine/Crawl/Snapshot/Persona admin forms), the plugin config grid,
+    REST API responses, and any future surface that round-trips raw config
+    values all agree on which keys to redact.
+    """
+    key = str(key or "")
+    upper = key.upper()
+    return key in _plugin_sensitive_config_keys() or any(needle in upper for needle in _SENSITIVE_CONFIG_KEY_NEEDLES)
+
+
+def redact_sensitive_config(config: Mapping[str, Any] | None) -> dict[str, Any]:
+    """Return a copy of ``config`` with credential values replaced by ``********``.
+
+    Used wherever a config dict crosses an API/export/debug-dump boundary. The
+    widget-side write-only treatment handles the form-render path; this helper
+    handles every JSON-response path (REST schemas, ``to_json`` exports, admin
+    debug views, etc.). Empty values are passed through unchanged so callers
+    can still tell "unset" from "set-but-hidden."
+    """
+    if config is None:
+        return {}
+    if not isinstance(config, Mapping):
+        return {}
+    redacted: dict[str, Any] = {}
+    for key, value in config.items():
+        if is_sensitive_config_key(str(key)) and value not in (None, ""):
+            redacted[key] = SENSITIVE_CONFIG_VALUE_REDACTED
+        else:
+            redacted[key] = value
+    return redacted
+
+
+def normalize_runtime_config(config: BaseConfigSet | Mapping[str, Any] | str | None) -> dict[str, Any]:
+    """Return a JSON-safe config dict suitable for storage or event payloads."""
+    if config is None:
+        return {}
+    if isinstance(config, BaseConfigSet):
+        config = config.model_dump(mode="json")
+    elif isinstance(config, str):
+        config = json.loads(config)
+    else:
+        config = dict(config)
+    return {key: value for key, value in json.loads(json.dumps(config, default=str)).items() if value is not None}
+
+
+def build_crawl_config_snapshot(
+    *,
+    user: Any = None,
+    persona: Any = None,
+    overrides: Mapping[str, Any] | None = None,
+    base_config: ArchiveBoxBaseConfig | Mapping[str, object] | None = None,
+) -> dict[str, Any]:
+    """Build the frozen runtime config stored on Crawl.config at creation time."""
+    effective = get_config(user=user, persona=persona, base_config=base_config)
+    frozen = effective.for_crawl_frozen()
+    if overrides:
+        frozen = get_config(base_config=frozen, overrides=overrides, include_machine=False).for_crawl_frozen()
+    return frozen
+
+
 def rprint(*args, file=None, **kwargs):
     console = _STDERR_CONSOLE if file is sys.stderr else _STDOUT_CONSOLE
     console.print(*args, **kwargs)
@@ -42,6 +153,7 @@ def rprint(*args, file=None, **kwargs):
 
 class ShellConfig(BaseConfigSet):
     toml_section_header: str = "SHELL_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_CRAWL_EXECUTION)
 
     DEBUG: bool = Field(default="--debug" in sys.argv)
 
@@ -73,6 +185,7 @@ class ShellConfig(BaseConfigSet):
 
 class StorageConfig(BaseConfigSet):
     toml_section_header: str = "STORAGE_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     # ARCHIVE_DIR / USERS_DIR are resolved dynamically via get_config().
     ARCHIVE_DIR: Path = Field(default=CONSTANTS.ARCHIVE_DIR)
@@ -82,37 +195,36 @@ class StorageConfig(BaseConfigSet):
     # TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
     # must be a short path due to unix path length restrictions for socket files (<100 chars)
     # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
-    TMP_DIR: Path = Field(default=CONSTANTS.DEFAULT_TMP_DIR)
+    TMP_DIR: Path = Field(default=CONSTANTS.DEFAULT_TMP_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
     # LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
     # must be able to contain executable binaries (up to 5GB size)
     # should not be a remote/network/FUSE mount for speed reasons, otherwise extractors will be slow
-    LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_DIR)
+    LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
-    # LIB_BIN_DIR is where installed binaries can be symlinked for shared runtime lookup.
-    # abxpkg/abx-dl build the executable lookup env at exec time.
-    LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR)
+    # LIB_BIN_DIR is an optional human-facing symlink convenience directory.
+    # Runtime lookup must use provider-specific paths under LIB_DIR instead.
+    LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
     # CUSTOM_TEMPLATES_DIR allows users to override default templates
     # defaults to DATA_DIR / 'user_templates' but can be configured
     CUSTOM_TEMPLATES_DIR: Path = Field(default=CONSTANTS.CUSTOM_TEMPLATES_DIR)
 
     OUTPUT_PERMISSIONS: str = Field(default="644")
-    RESTRICT_FILE_NAMES: str = Field(default="windows")
     ENFORCE_ATOMIC_WRITES: bool = Field(default=True)
-
-    # not supposed to be user settable:
-    DIR_OUTPUT_PERMISSIONS: str = Field(default="755")  # computed from OUTPUT_PERMISSIONS
+    ALLOW_NO_UNIX_SOCKETS: bool = Field(default=False, alias="ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS")
 
 
 class GeneralConfig(BaseConfigSet):
     toml_section_header: str = "GENERAL_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     TAG_SEPARATOR_PATTERN: str = Field(default=r"[,]")
 
 
 class ServerConfig(BaseConfigSet):
     toml_section_header: str = "SERVER_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     SERVER_SECURITY_MODES: ClassVar[tuple[str, ...]] = (
         "safe-subdomains-fullreplay",
@@ -123,23 +235,18 @@ class ServerConfig(BaseConfigSet):
 
     SECRET_KEY: str = Field(default_factory=lambda: "".join(secrets.choice("abcdefghijklmnopqrstuvwxyz0123456789_") for _ in range(50)))
     BIND_ADDR: str = Field(default="127.0.0.1:8000")
-    LISTEN_HOST: str = Field(default="archivebox.localhost:8000")
-    ADMIN_BASE_URL: str = Field(default="")
-    ARCHIVE_BASE_URL: str = Field(default="")
+    BASE_URL: str = Field(default="")
     ALLOWED_HOSTS: str = Field(default="*")
-    CSRF_TRUSTED_ORIGINS: str = Field(default="http://admin.archivebox.localhost:8000")
+    CSRF_TRUSTED_ORIGINS: str = Field(default="")
     SERVER_SECURITY_MODE: str = Field(default="safe-subdomains-fullreplay")
 
-    SNAPSHOTS_PER_PAGE: int = Field(default=40)
-    PREVIEW_ORIGINALS: bool = Field(default=True)
+    SNAPSHOTS_PER_PAGE: int = Field(default=50, ge=1)
     FOOTER_INFO: str = Field(
         default="Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.",
     )
     # CUSTOM_TEMPLATES_DIR: Path          = Field(default=None)  # this is now a constant
 
     PUBLIC_INDEX: bool = Field(default=True)
-    PUBLIC_SNAPSHOTS: bool = Field(default=True)
-    PUBLIC_SNAPSHOTS_LIST: bool | None = Field(default=None)
     PUBLIC_ADD_VIEW: bool = Field(default=False)
 
     ADMIN_USERNAME: str | None = Field(default=None)
@@ -196,63 +303,42 @@ class ServerConfig(BaseConfigSet):
         )
 
 
-def _print_server_security_mode_warning(config: ServerConfig) -> None:
-    if not config.IS_LOWER_SECURITY_MODE:
-        return
-    if config.SERVER_SECURITY_MODE in _WARNED_SERVER_SECURITY_MODES:
-        return
+class DatabaseConfig(BaseConfigSet):
+    toml_section_header: str = "DATABASE_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
-    rprint(
-        f"[yellow][!] WARNING: ArchiveBox is running with SERVER_SECURITY_MODE={config.SERVER_SECURITY_MODE}[/yellow]",
-        file=sys.stderr,
+    DATABASE_NAME: str = Field(default=str(CONSTANTS.DATABASE_FILE), alias="ARCHIVEBOX_DATABASE_NAME")
+    SQLITE_JOURNAL_MODE: str = Field(
+        default="WAL",
+        alias="ARCHIVEBOX_SQLITE_JOURNAL_MODE",
+        pattern=r"(?i)^(DELETE|TRUNCATE|PERSIST|MEMORY|WAL|OFF)$",
     )
-    rprint(
-        "[yellow]    Archived pages may share an origin with privileged app routes in this mode.[/yellow]",
-        file=sys.stderr,
+    SQLITE_MMAP_SIZE: int = Field(
+        default=0 if CONSTANTS.IN_DOCKER else 134217728,
+        alias="ARCHIVEBOX_SQLITE_MMAP_SIZE",
+        ge=0,
     )
-    rprint(
-        "[yellow]    To switch to the safer isolated setup:[/yellow]",
-        file=sys.stderr,
-    )
-    rprint(
-        "[yellow]    1. Set SERVER_SECURITY_MODE=safe-subdomains-fullreplay[/yellow]",
-        file=sys.stderr,
-    )
-    rprint(
-        "[yellow]    2. Point *.archivebox.localhost (or your chosen base domain) at this server[/yellow]",
-        file=sys.stderr,
-    )
-    rprint(
-        "[yellow]    3. Configure wildcard DNS/TLS or your reverse proxy so admin., web., api., and snapshot subdomains resolve[/yellow]",
-        file=sys.stderr,
-    )
-    _WARNED_SERVER_SECURITY_MODES.add(config.SERVER_SECURITY_MODE)
+    SQLITE_BUSY_TIMEOUT: int = Field(default=30000, alias="ARCHIVEBOX_SQLITE_BUSY_TIMEOUT", ge=0)
+    SQLITE_LOCK_RETRY_TIMEOUT: float = Field(default=60.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_TIMEOUT", ge=0)
+    SQLITE_LOCK_RETRY_INTERVAL: float = Field(default=5.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_INTERVAL", gt=0)
 
 
 class ArchivingConfig(BaseConfigSet):
     toml_section_header: str = "ARCHIVING_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_CRAWL_FROZEN)
 
     PLUGINS: str = Field(
         default="",
         description="Comma-separated plugin selection for this run. Empty means use enabled plugin defaults.",
     )
-    ENABLED_PLUGINS: str = Field(
-        default="",
-        description="Comma-separated plugin selection override used by the UI and API.",
-    )
-    ENABLED_EXTRACTORS: str = Field(
-        default="",
-        description="Legacy comma-separated plugin selection override.",
-    )
 
     ONLY_NEW: bool = Field(default=True)
-    OVERWRITE: bool = Field(default=False)
+    INDEX_ONLY: bool = Field(default=False)
 
     TIMEOUT: int = Field(default=60)
-    MAX_URL_ATTEMPTS: int = Field(default=50)
-    MAX_DEPTH: int = Field(default=0)
     CRAWL_MAX_URLS: int = Field(default=0)
     CRAWL_MAX_SIZE: int = Field(default=0)
+    CRAWL_TIMEOUT: int = Field(default=0, description="Maximum total crawl runtime in seconds (0 = unlimited).")
     CRAWL_MAX_CONCURRENT_SNAPSHOTS: int = Field(
         default=4,
         description="Maximum number of snapshots to archive concurrently within one crawl.",
@@ -269,10 +355,19 @@ class ArchivingConfig(BaseConfigSet):
     URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST")
     URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST")
 
-    SAVE_ALLOWLIST: dict[str, list[str]] = Field(default={})  # mapping of regex patterns to list of archive methods
-    SAVE_DENYLIST: dict[str, list[str]] = Field(default={})
-
     DEFAULT_PERSONA: str = Field(default="Default")
+    PERMISSIONS: str = Field(
+        default="public",
+        description="Snapshot visibility: public lists and serves content, unlisted serves direct links only, private requires admin login.",
+    )
+    DELETE_AFTER: str = Field(
+        default="0",
+        description=(
+            "Automatically delete Crawl, Snapshot, ArchiveResult, and Process rows after this duration. "
+            "Use 0, '', or None to disable. Allowed units: h/hr/hour, d/day, w/week, mo/month, y/year; "
+            "minimum non-zero duration is 1h."
+        ),
+    )
 
     def warn_if_invalid(self) -> None:
         if int(self.TIMEOUT) < 5:
@@ -293,6 +388,22 @@ class ArchivingConfig(BaseConfigSet):
             urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
         return v
 
+    @field_validator("DELETE_AFTER", mode="before")
+    @classmethod
+    def validate_delete_after(cls, value):
+        parse_delete_after(value)
+        if value is None:
+            return "0"
+        return str(value).strip() or "0"
+
+    @field_validator("PERMISSIONS", mode="before")
+    @classmethod
+    def validate_permissions(cls, value):
+        normalized = str(value or "public").strip().lower()
+        if normalized not in {"public", "unlisted", "private"}:
+            raise ValueError("PERMISSIONS must be one of: public, unlisted, private.")
+        return normalized
+
     @property
     def URL_ALLOWLIST_PTN(self) -> re.Pattern | None:
         return re.compile(self.URL_ALLOWLIST, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS) if self.URL_ALLOWLIST else None
@@ -301,39 +412,44 @@ class ArchivingConfig(BaseConfigSet):
     def URL_DENYLIST_PTN(self) -> re.Pattern:
         return re.compile(self.URL_DENYLIST, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS)
 
-    @property
-    def SAVE_ALLOWLIST_PTNS(self) -> dict[re.Pattern, list[str]]:
-        return (
-            {
-                # regexp: methods list
-                re.compile(key, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): val
-                for key, val in self.SAVE_ALLOWLIST.items()
-            }
-            if self.SAVE_ALLOWLIST
-            else {}
-        )
 
-    @property
-    def SAVE_DENYLIST_PTNS(self) -> dict[re.Pattern, list[str]]:
-        return (
-            {
-                # regexp: methods list
-                re.compile(key, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): val
-                for key, val in self.SAVE_DENYLIST.items()
-            }
-            if self.SAVE_DENYLIST
-            else {}
-        )
+def parse_delete_after(value) -> timedelta | None:
+    if value is None:
+        return None
+
+    raw = str(value).strip().lower()
+    if raw in ("", "0", "none", "false", "no", "off"):
+        return None
+
+    match = re.fullmatch(r"(\d+)\s*(h|hr|hrs|hour|hours|d|day|days|w|week|weeks|mo|month|months|y|yr|yrs|year|years)", raw)
+    if not match:
+        raise ValueError("DELETE_AFTER must be 0 or a duration like 1h, 7d, 4w, 6mo, or 1y.")
+
+    amount = int(match.group(1))
+    unit = match.group(2)
+    if amount <= 0:
+        return None
+    if unit in ("h", "hr", "hrs", "hour", "hours"):
+        duration = timedelta(hours=amount)
+    elif unit in ("d", "day", "days"):
+        duration = timedelta(days=amount)
+    elif unit in ("w", "week", "weeks"):
+        duration = timedelta(weeks=amount)
+    elif unit in ("mo", "month", "months"):
+        duration = timedelta(days=30 * amount)
+    else:
+        duration = timedelta(days=365 * amount)
+
+    if duration < timedelta(hours=1):
+        raise ValueError("DELETE_AFTER must be 0 or at least 1h.")
+    return duration
 
 
 class SearchBackendConfig(BaseConfigSet):
     toml_section_header: str = "SEARCH_BACKEND_CONFIG"
-
-    USE_INDEXING_BACKEND: bool = Field(default=True)
-    USE_SEARCHING_BACKEND: bool = Field(default=True)
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep")
-    SEARCH_PROCESS_HTML: bool = Field(default=True)
 
 
 def _plugin_user_config_value(value: Any) -> str:
@@ -349,7 +465,7 @@ def _plugin_user_config(config: Mapping[str, object]) -> dict[str, str]:
 
 
 def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
 
     schemas: PluginSchemaDocuments = {}
     if BASE_CONFIG_PATH.exists():
@@ -373,6 +489,7 @@ def _plugin_config_model(plugin_schemas: PluginSchemaDocuments) -> type[BaseMode
     return build_config_model("ArchiveBoxPluginConfig", _plugin_config_properties(plugin_schemas))
 
 
+@lru_cache(maxsize=1)
 def _archivebox_config_input_names() -> set[str]:
     names = set(ArchiveBoxConfig.model_fields)
     for field in ArchiveBoxConfig.model_fields.values():
@@ -386,6 +503,7 @@ class ArchiveBoxBaseConfig(
     StorageConfig,
     GeneralConfig,
     ServerConfig,
+    DatabaseConfig,
     ArchivingConfig,
     SearchBackendConfig,
     LDAPConfig,
@@ -406,13 +524,78 @@ class ArchiveBoxBaseConfig(
         populate_by_name=True,
     )
 
-    DATA_DIR: Path = Field(default=CONSTANTS.DATA_DIR)
-    ABX_RUNTIME: str = Field(default="archivebox")
-    CRAWL_DIR: Path | None = Field(default=None)
-    CRAWL_OUTPUT_DIR: Path | None = Field(default=None)
-    SNAP_DIR: Path | None = Field(default=None)
+    DATA_DIR: Path = Field(default=CONSTANTS.DATA_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    ABX_RUNTIME: str = Field(default="archivebox", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    CRAWL_DIR: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    SNAP_DIR: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     computed_config_keys: ClassVar[tuple[str, ...]] = COMPUTED_CONFIG_KEYS
 
+    @classmethod
+    def _core_config_classes(cls) -> tuple[type[BaseConfigSet], ...]:
+        return (
+            ShellConfig,
+            StorageConfig,
+            GeneralConfig,
+            ServerConfig,
+            DatabaseConfig,
+            ArchivingConfig,
+            SearchBackendConfig,
+            LDAPConfig,
+        )
+
+    @classmethod
+    def _core_field_scope(cls, key: str) -> str | None:
+        if key == "toml_section_header":
+            return _SCOPE_SERVER
+        for config_cls in cls._core_config_classes():
+            field = config_cls.model_fields.get(key)
+            if field is None:
+                continue
+            default_scope = str(config_cls.__private_attributes__["_scope"].default)
+            extra = field.json_schema_extra
+            if isinstance(extra, dict) and "scope" in extra:
+                return str(extra["scope"])
+            return default_scope
+        if key in ArchiveBoxBaseConfig.model_fields:
+            field = ArchiveBoxBaseConfig.model_fields[key]
+            extra = field.json_schema_extra
+            if isinstance(extra, dict) and "scope" in extra:
+                return str(extra["scope"])
+            return _SCOPE_SERVER
+        return None
+
+    @classmethod
+    def _plugin_field_scope(cls, key: str) -> str | None:
+        for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+            properties = schema.get("properties") if isinstance(schema, dict) else None
+            if not isinstance(properties, dict) or key not in properties:
+                continue
+            prop_schema = properties.get(key) or {}
+            if isinstance(prop_schema, Mapping) and prop_schema.get("x-scope"):
+                return str(prop_schema["x-scope"])
+            if str(plugin_name).startswith("search_backend_"):
+                return _SCOPE_SERVER
+            return _SCOPE_CRAWL_FROZEN
+        return None
+
+    @classmethod
+    def scope_for_key(cls, key: str) -> str:
+        return cls._core_field_scope(key) or cls._plugin_field_scope(key) or _SCOPE_SERVER
+
+    def _scoped_config(self, *, include_execution: bool) -> dict[str, Any]:
+        allowed_scopes = {_SCOPE_CRAWL_FROZEN}
+        if include_execution:
+            allowed_scopes.add(_SCOPE_CRAWL_EXECUTION)
+        return {key: value for key, value in normalize_runtime_config(self).items() if type(self).scope_for_key(key) in allowed_scopes}
+
+    def for_crawl_execution(self) -> dict[str, Any]:
+        """Config safe to pass to crawl/snapshot hook execution."""
+        return self._scoped_config(include_execution=True)
+
+    def for_crawl_frozen(self) -> dict[str, Any]:
+        """Config safe to persist permanently on Crawl.config."""
+        return self._scoped_config(include_execution=False)
+
     @model_validator(mode="after")
     def resolve_runtime_paths(self):
         self.DATA_DIR = self.DATA_DIR.expanduser().resolve()
@@ -469,6 +652,7 @@ ArchiveBoxConfig = _build_archivebox_config_model(PLUGIN_CONFIG_SCHEMAS)
 def get_config(
     defaults: ConfigOverrides | None = None,
     overrides: ConfigOverrides | None = None,
+    base_config: ArchiveBoxBaseConfig | Mapping[str, object] | None = None,
     persona: Any = None,
     user: Any = None,
     crawl: Any = None,
@@ -477,21 +661,23 @@ def get_config(
     machine: Any = None,
     include_machine: bool = True,
     resolve_plugins: bool = True,
+    redact_sensitive: bool = False,
 ) -> ArchiveBoxBaseConfig:
     """
     Get merged config from all sources.
 
     Priority (highest to lowest):
     1. Explicit overrides
-    2. Per-snapshot config and output path
-    3. Per-crawl config and output path
-    4. Per-user config
-    5. Per-persona derived config
-    6. Current machine derived config
-    7. Environment variables
-    8. Config file (ArchiveBox.conf)
-    9. Plugin schema defaults
-    10. Core config defaults
+    2. Per-ArchiveResult config
+    3. Per-snapshot config and output path
+    4. Frozen per-crawl config and output path
+    5. Per-user config (only when resolving outside a crawl)
+    6. Per-persona derived config (only when resolving outside a crawl)
+    7. Current machine derived config (only when resolving outside a crawl)
+    8. Environment variables (only when resolving outside a crawl)
+    9. Config file (ArchiveBox.conf, only when resolving outside a crawl)
+    10. Plugin schema defaults
+    11. Core config defaults
     """
     if snapshot is None and archiveresult is not None:
         snapshot = archiveresult.snapshot
@@ -499,7 +685,9 @@ def get_config(
     if crawl is None and snapshot is not None:
         crawl = snapshot.crawl
 
-    if include_machine and machine is None:
+    crawl_config_base = crawl is not None and base_config is None
+
+    if include_machine and machine is None and not crawl_config_base:
         try:
             from django.apps import apps
 
@@ -510,48 +698,46 @@ def get_config(
         except Exception:
             machine = None
 
-    if persona is None and crawl is not None:
-        from archivebox.personas.models import Persona
-
-        persona_id = crawl.persona_id
-        if persona_id:
-            persona = Persona.objects.filter(id=persona_id).first()
-            if persona is None:
-                raise Persona.DoesNotExist(f"Crawl {crawl.id} references missing Persona {persona_id}")
-
-        if persona is None:
-            crawl_config = crawl.config or {}
-            default_persona_name = str(crawl_config.get("DEFAULT_PERSONA") or "").strip()
-            if default_persona_name:
-                persona, _ = Persona.objects.get_or_create(name=default_persona_name or "Default")
-                persona.ensure_dirs()
+    if persona is None and crawl is not None and not crawl_config_base:
+        persona = crawl.resolve_persona()
 
     config_data: ConfigPayload = dict(defaults or {})
-    config_data.update(ArchiveBoxConfig().model_dump(mode="json"))
-
-    plugin_schemas = {
-        plugin_name: schema.get("properties", {}) for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)
-    }
+    base_config_payload: ConfigPayload = {}
+    if crawl_config_base:
+        config_data.update(dict(crawl.config or {}))
+    elif base_config is not None:
+        if isinstance(base_config, ArchiveBoxBaseConfig):
+            base_config_payload.update(base_config.model_dump(mode="json"))
+        else:
+            base_config_payload.update(dict(base_config))
+        config_data.update(base_config_payload)
+    else:
+        config_data.update(ArchiveBoxConfig().model_dump(mode="json"))
+        legacy_permissions = permissions_from_legacy_public_flags({**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ})
+        if legacy_permissions:
+            config_data["PERMISSIONS"] = legacy_permissions
 
     scope_overrides: ConfigPayload = {}
 
-    if include_machine and machine is not None and machine.config:
-        from archivebox.machine.models import _sanitize_machine_config
+    if not crawl_config_base:
+        if include_machine and machine is not None and machine.config:
+            from archivebox.machine.models import _sanitize_machine_config
 
-        scope_overrides.update(_sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")))
+            scope_overrides.update(_sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")))
 
-    if persona is not None:
-        scope_overrides.update(persona.get_derived_config())
+        if persona is not None:
+            scope_overrides.update(persona.get_derived_config())
 
-    if user is not None and user.config:
-        scope_overrides.update(user.config)
+        user_config = getattr(user, "config", None)
+        if user_config:
+            scope_overrides.update(user_config)
 
-    if crawl is not None and crawl.config:
+    if crawl is not None and crawl.config and not crawl_config_base:
         scope_overrides.update(crawl.config)
 
     if crawl is not None:
-        scope_overrides["CRAWL_OUTPUT_DIR"] = crawl.output_dir
-        scope_overrides["CRAWL_DIR"] = crawl.output_dir
+        if not overrides or "CRAWL_DIR" not in overrides:
+            scope_overrides["CRAWL_DIR"] = crawl.output_dir
 
     if snapshot is not None and snapshot.config:
         scope_overrides.update(snapshot.config)
@@ -559,26 +745,71 @@ def get_config(
     if snapshot is not None:
         scope_overrides["SNAP_DIR"] = snapshot.output_dir
 
+    if archiveresult is not None and archiveresult.config:
+        scope_overrides.update(archiveresult.config)
+
     if overrides:
         scope_overrides.update(overrides)
 
+    legacy_scope_permissions = permissions_from_legacy_public_flags(scope_overrides)
+    if legacy_scope_permissions:
+        scope_overrides["PERMISSIONS"] = legacy_scope_permissions
+
     archivebox_scope_overrides = {key: value for key, value in scope_overrides.items() if key in _archivebox_config_input_names()}
     config_data.update(archivebox_scope_overrides)
 
     if resolve_plugins:
+        plugin_schemas = {
+            plugin_name: schema.get("properties", {}) for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)
+        }
         plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
+        plugin_user_config = _plugin_user_config(scope_overrides)
+        if not crawl_config_base:
+            plugin_user_config = {**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **plugin_user_config}
         plugin_sections = resolve_plugin_configs(
             plugin_schemas,
             global_config=plugin_global_config,
-            user_config={**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **_plugin_user_config(scope_overrides)},
+            user_config=plugin_user_config,
         )
         for plugin_config in plugin_sections.values():
             config_data.update(plugin_config)
+        if base_config_payload:
+            config_data.update({key: value for key, value in base_config_payload.items() if key in _archivebox_config_input_names()})
+        if crawl_config_base:
+            config_data.update(dict(crawl.config or {}))
         config_data.update(archivebox_scope_overrides)
 
     config_data["ABX_RUNTIME"] = "archivebox"
 
+    # Decode JSON-encoded complex values (dict/list fields) that came from
+    # string-only sources before validation. ``IniConfigSettingsSource`` does
+    # this for the ArchiveBox.conf path, but Machine.config (mirrored from the
+    # INI via ``_coerce_to_str_dict``) and plugin/env scope overrides bypass
+    # pydantic-settings sources entirely — they feed JSON strings directly
+    # into ``model_validate``, which rejects ``"{...}"`` for a ``dict[str, str]``
+    # field. Run pydantic-settings' own complex-value decoder here so every
+    # source converges on the same shape before validation.
+    _complex_decoder = IniConfigSettingsSource(ArchiveBoxConfig)
+    for _field_name, _field in ArchiveBoxConfig.model_fields.items():
+        if _field_name not in config_data:
+            continue
+        _raw = config_data[_field_name]
+        if not isinstance(_raw, str) or not _raw:
+            continue
+        if _complex_decoder.field_is_complex(_field):
+            config_data[_field_name] = _complex_decoder.prepare_field_value(
+                _field_name,
+                _field,
+                _raw,
+                True,
+            )
+
     config = ArchiveBoxConfig.model_validate(config_data)
+    if redact_sensitive:
+        for key in type(config).model_fields:
+            value = getattr(config, key, None)
+            if is_sensitive_config_key(key) and value not in (None, ""):
+                setattr(config, key, SENSITIVE_CONFIG_VALUE_REDACTED)
     os.environ["LIB_DIR"] = str(config.LIB_DIR)
     os.environ["LIB_BIN_DIR"] = str(config.LIB_BIN_DIR)
     os.environ["ABXPKG_LIB_DIR"] = str(config.LIB_DIR)
@@ -586,7 +817,6 @@ def get_config(
     if archiving_warning_key not in _WARNED_ARCHIVING_CONFIGS:
         config.warn_if_invalid()
         _WARNED_ARCHIVING_CONFIGS.add(archiving_warning_key)
-    _print_server_security_mode_warning(config)
     return config
 
 
@@ -597,6 +827,7 @@ def get_all_configs() -> dict[str, BaseConfigSet]:
         "STORAGE_CONFIG": StorageConfig(),
         "GENERAL_CONFIG": GeneralConfig(),
         "SERVER_CONFIG": ServerConfig(),
+        "DATABASE_CONFIG": DatabaseConfig(),
         "ARCHIVING_CONFIG": ArchivingConfig(),
         "SEARCH_BACKEND_CONFIG": SearchBackendConfig(),
         "LDAP_CONFIG": LDAPConfig(),

archivebox/config/configset.py

@@ -22,8 +22,6 @@ COMPUTED_CONFIG_KEYS = (
     "IS_LOWER_SECURITY_MODE",
     "URL_ALLOWLIST_PTN",
     "URL_DENYLIST_PTN",
-    "SAVE_ALLOWLIST_PTNS",
-    "SAVE_DENYLIST_PTNS",
 )
 
 
@@ -32,6 +30,33 @@ class CaseConfigParser(ConfigParser):
         return optionstr
 
 
+# ``IniConfigSettingsSource.get_field_value`` is called once per pydantic field
+# (hundreds of fields across the GlobalConfig + plugin configs). Without
+# caching, every field re-opens and re-parses ``ArchiveBox.conf`` from disk —
+# that's the dominant cost of ``get_config()`` (each call was ~130ms; profile
+# showed 632 ``parser.read(config_path)`` invocations per call). The cache is
+# keyed on (path, mtime) so external edits to the file still get picked up.
+_INI_CACHE: dict[tuple[str, float], dict[str, Any]] = {}
+
+
+def _read_ini_config_cached(config_path_str: str) -> dict[str, Any]:
+    config_path = Path(config_path_str)
+    try:
+        mtime = config_path.stat().st_mtime
+    except FileNotFoundError:
+        return {}
+    cache_key = (str(config_path), mtime)
+    cached = _INI_CACHE.get(cache_key)
+    if cached is not None:
+        return cached
+    parser = CaseConfigParser()
+    parser.read(config_path)
+    flat = {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+    _INI_CACHE.clear()
+    _INI_CACHE[cache_key] = flat
+    return flat
+
+
 class IniConfigSettingsSource(PydanticBaseSettingsSource):
     """
     Custom settings source that reads from ArchiveBox.conf (INI format).
@@ -41,10 +66,27 @@ class IniConfigSettingsSource(PydanticBaseSettingsSource):
     def get_field_value(self, field: Any, field_name: str) -> tuple[Any, str, bool]:
         config_vals = self._load_config_file()
         field_value = config_vals.get(field_name.upper())
-        return field_value, field_name, False
+        # Mark complex-typed fields (``dict``/``list``) so the parent
+        # ``prepare_field_value`` JSON-decodes the INI string before pydantic
+        # validates against the dict/list annotation. Without this, e.g.
+        # ``ABX_INSTALL_CACHE`` arrives as a raw JSON string and pydantic
+        # rejects it with ``Input should be a valid dictionary``.
+        value_is_complex = bool(field_value is not None and self.field_is_complex(field))
+        return field_value, field_name, value_is_complex
 
     def __call__(self) -> dict[str, Any]:
-        return self._load_config_file()
+        # Use the per-field path (``get_field_value`` + ``prepare_field_value``)
+        # so complex types get JSON-decoded. The previous flat-dict return
+        # skipped pydantic-settings' complex-value handling entirely.
+        result: dict[str, Any] = {}
+        for field_name, field in self.settings_cls.model_fields.items():
+            value, key, value_is_complex = self.get_field_value(field, field_name)
+            if value is None:
+                continue
+            prepared = self.prepare_field_value(field_name, field, value, value_is_complex)
+            if prepared is not None:
+                result[key] = prepared
+        return result
 
     def _load_config_file(self) -> dict[str, Any]:
         try:
@@ -54,14 +96,7 @@ class IniConfigSettingsSource(PydanticBaseSettingsSource):
         except ImportError:
             return {}
 
-        if not config_path.exists():
-            return {}
-
-        parser = CaseConfigParser()
-        parser.read(config_path)
-
-        # Flatten all sections into single namespace (ignore section headers)
-        return {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+        return _read_ini_config_cached(str(config_path))
 
 
 class BaseConfigSet(BaseSettings):

archivebox/config/constants.py

@@ -17,10 +17,9 @@ import sys
 
 from pathlib import Path
 
-from benedict import benedict
 from platformdirs import user_config_path
 
-from archivebox.misc.logging import DEFAULT_CLI_COLORS
+from archivebox.misc.logging import AttrDict, DEFAULT_CLI_COLORS
 
 from .paths import (
     PACKAGE_DIR,
@@ -134,7 +133,7 @@ class ConstantsDict:
     # Config constants
     TIMEZONE: str = "UTC"
     DEFAULT_CLI_COLORS: dict[str, str] = DEFAULT_CLI_COLORS
-    DISABLED_CLI_COLORS: dict[str, str] = benedict({k: "" for k in DEFAULT_CLI_COLORS})
+    DISABLED_CLI_COLORS: dict[str, str] = AttrDict({k: "" for k in DEFAULT_CLI_COLORS})
 
     # Hard safety limits (seconds)
     MAX_HOOK_RUNTIME_SECONDS: int = 60 * 60 * 12  # 12 hours
@@ -291,14 +290,9 @@ class ConstantsDict:
         # so it behaves like a dict[key] == dict.key or object attr
         return getattr(cls, key)
 
-    @classmethod
-    def __benedict__(cls):
-        # when casting to benedict, only include uppercase keys that don't start with an underscore
-        return benedict({key: value for key, value in cls.__dict__.items() if key.isupper() and not key.startswith("_")})
-
 
 CONSTANTS = ConstantsDict
-CONSTANTS_CONFIG = CONSTANTS.__benedict__()
+CONSTANTS_CONFIG = AttrDict({key: value for key, value in CONSTANTS.__dict__.items() if key.isupper() and not key.startswith("_")})
 
 # add all key: values to globals() for easier importing, e.g.:
 # from archivebox.config.constants import IS_ROOT, PERSONAS_DIR, ...

archivebox/config/django.py

@@ -28,14 +28,6 @@ STDERR = Console(stderr=True)
 logging.CONSOLE = CONSOLE
 
 
-def setup_django_minimal():
-    # sys.path.append(str(CONSTANTS.PACKAGE_DIR))
-    # os.environ.setdefault('ARCHIVEBOX_DATA_DIR', str(CONSTANTS.DATA_DIR))
-    # os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
-    # django.setup()
-    raise Exception("dont use this anymore")
-
-
 DJANGO_SET_UP = False
 
 
@@ -49,6 +41,11 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
         # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
         return
 
+    # Third-party patches are only needed once Django/apps are about to load.
+    # Keeping them out of archivebox.__init__ avoids paying Django/Daphne setup
+    # cost for cheap CLI startup paths like `archivebox <cmd> --help`.
+    import archivebox.misc.monkey_patches  # noqa: F401
+
     from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, SudoPermission
 
     # if running as root, chown the data dir to the archivebox user to make sure it's accessible to the archivebox user
@@ -165,6 +162,6 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
             #     logfire.info(f'Started ArchiveBox v{CONSTANTS.VERSION}', argv=sys.argv)
 
     except KeyboardInterrupt:
-        raise SystemExit(2)
+        raise
 
     DJANGO_SET_UP = True

archivebox/config/ldap.py

@@ -1,6 +1,6 @@
 __package__ = "archivebox.config"
 
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 
 from archivebox.config.configset import BaseConfigSet
 
@@ -14,6 +14,7 @@ class LDAPConfig(BaseConfigSet):
     """
 
     toml_section_header: str = "LDAP_CONFIG"
+    _scope: str = PrivateAttr(default="server")
 
     LDAP_ENABLED: bool = Field(default=False)
     LDAP_SERVER_URI: str | None = Field(default=None)

archivebox/config/paths.py

@@ -11,8 +11,6 @@ from functools import cache
 from datetime import datetime
 from typing import TYPE_CHECKING
 
-from benedict import benedict
-
 from .permissions import SudoPermission, IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
 
 if TYPE_CHECKING:
@@ -262,6 +260,7 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
 def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs):
     from archivebox.config.constants import CONSTANTS
     from archivebox.config.common import get_config
+    from archivebox.misc.logging import AttrDict
 
     config = config or get_config(**config_kwargs)
     try:
@@ -269,7 +268,7 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
     except Exception:
         tmp_dir = config.TMP_DIR
 
-    return benedict(
+    return AttrDict(
         {
             "DATA_DIR": {
                 "path": DATA_DIR.resolve(),
@@ -344,6 +343,7 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
 def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs):
     from archivebox.config.constants import CONSTANTS
     from archivebox.config.common import get_config
+    from archivebox.misc.logging import AttrDict
 
     config = config or get_config(**config_kwargs)
     try:
@@ -353,7 +353,7 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
 
     lib_bin_dir = lib_dir / "bin"
 
-    return benedict(
+    return AttrDict(
         {
             "PACKAGE_DIR": {
                 "path": (PACKAGE_DIR).resolve(),

archivebox/config/permissions.py

@@ -15,6 +15,8 @@ from contextlib import contextmanager
 #############################################################################################
 
 DATA_DIR = Path(os.getcwd())
+if os.environ.get("DATA_DIR") and Path(os.environ["DATA_DIR"]).resolve() != DATA_DIR.resolve():
+    raise SystemExit(f"[X] DATA_DIR={os.environ['DATA_DIR']} must equal cwd={DATA_DIR}; cd into the data dir before running archivebox")
 
 try:
     DATA_DIR_STAT = DATA_DIR.stat()

archivebox/config/views.py

@@ -1,13 +1,9 @@
 __package__ = "archivebox.config"
 
-import html
-import json
 import os
 import inspect
-import re
 from pathlib import Path
 from typing import Any
-from collections.abc import Callable
 from urllib.parse import quote, urlencode
 from django.http import HttpRequest
 from django.utils import timezone
@@ -22,8 +18,6 @@ from archivebox.misc.util import parse_date
 
 from archivebox.machine.models import Binary
 
-ABX_PLUGINS_DOCS_BASE_URL = "https://archivebox.github.io/abx-plugins/"
-ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/"
 LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/"
 INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"
@@ -38,55 +32,6 @@ def format_parsed_datetime(value: object) -> str:
     return parsed.strftime("%Y-%m-%d %H:%M:%S") if parsed else ""
 
 
-JSON_TOKEN_RE = re.compile(
-    r'(?P<key>"(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*")(?=\s*:)'
-    r'|(?P<string>"(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*")'
-    r"|(?P<boolean>\btrue\b|\bfalse\b)"
-    r"|(?P<null>\bnull\b)"
-    r"|(?P<number>-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)",
-)
-
-
-def render_code_block(text: str, *, highlighted: bool = False) -> str:
-    code = html.escape(text, quote=False)
-
-    if highlighted:
-
-        def _wrap_token(match: re.Match[str]) -> str:
-            styles = {
-                "key": "color: #0550ae;",
-                "string": "color: #0a7f45;",
-                "boolean": "color: #8250df; font-weight: 600;",
-                "null": "color: #6e7781; font-style: italic;",
-                "number": "color: #b35900;",
-            }
-            token_type = next(name for name, value in match.groupdict().items() if value is not None)
-            return f'<span style="{styles[token_type]}">{match.group(0)}</span>'
-
-        code = JSON_TOKEN_RE.sub(_wrap_token, code)
-
-    return (
-        '<pre style="max-height: 600px; overflow: auto; background: #f6f8fa; '
-        'border: 1px solid #d0d7de; border-radius: 6px; padding: 12px; margin: 0;">'
-        '<code style="font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, '
-        "'Liberation Mono', monospace; white-space: pre; line-height: 1.5;\">"
-        f"{code}"
-        "</code></pre>"
-    )
-
-
-def render_highlighted_json_block(value: Any) -> str:
-    return render_code_block(json.dumps(value, indent=2, ensure_ascii=False), highlighted=True)
-
-
-def get_plugin_docs_url(plugin_name: str) -> str:
-    return f"{ABX_PLUGINS_DOCS_BASE_URL}#{plugin_name}"
-
-
-def get_plugin_hook_source_url(plugin_name: str, hook_name: str) -> str:
-    return f"{ABX_PLUGINS_GITHUB_BASE_URL}{quote(plugin_name)}/{quote(hook_name)}"
-
-
 def get_live_config_url(key: str) -> str:
     return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
 
@@ -104,194 +49,6 @@ def get_installed_binary_change_url(name: str, binary: Binary | None) -> str | N
     return f"{base_url}?{urlencode({'_changelist_filters': changelist_filters})}"
 
 
-def get_machine_admin_url() -> str | None:
-    try:
-        from archivebox.machine.models import Machine
-
-        return Machine.current().admin_change_url
-    except Exception:
-        return None
-
-
-def render_code_tag_list(values: list[str]) -> str:
-    if not values:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    tags = "".join(
-        str(
-            format_html(
-                '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>',
-                value,
-            ),
-        )
-        for value in values
-    )
-    return f'<div style="display: flex; flex-wrap: wrap;">{tags}</div>'
-
-
-def render_plugin_metadata_html(config: dict[str, Any]) -> str:
-    required_binaries = [
-        str(item.get("name")) for item in (config.get("required_binaries") or []) if isinstance(item, dict) and item.get("name")
-    ]
-    rows = (
-        ("Title", config.get("title") or "(none)"),
-        ("Description", config.get("description") or "(none)"),
-        ("Required Plugins", mark_safe(render_link_tag_list(config.get("required_plugins") or [], get_plugin_docs_url))),
-        ("Required Binaries", mark_safe(render_link_tag_list(required_binaries, get_environment_binary_url))),
-        ("Output MIME Types", mark_safe(render_code_tag_list(config.get("output_mimetypes") or []))),
-    )
-
-    rendered_rows = "".join(
-        str(
-            format_html(
-                '<div style="margin: 0 0 14px 0;"><div style="font-weight: 600; margin-bottom: 4px;">{}</div><div>{}</div></div>',
-                label,
-                value,
-            ),
-        )
-        for label, value in rows
-    )
-    return f'<div style="margin: 4px 0 0 0;">{rendered_rows}</div>'
-
-
-def render_link_tag_list(values: list[str], url_resolver: Callable[[str], str] | None = None) -> str:
-    if not values:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    tags = []
-    for value in values:
-        if url_resolver is None:
-            tags.append(
-                str(
-                    format_html(
-                        '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                        'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>',
-                        value,
-                    ),
-                ),
-            )
-        else:
-            tags.append(
-                str(
-                    format_html(
-                        '<a href="{}" style="text-decoration: none;">'
-                        '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                        'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>'
-                        "</a>",
-                        url_resolver(value),
-                        value,
-                    ),
-                ),
-            )
-    return f'<div style="display: flex; flex-wrap: wrap;">{"".join(tags)}</div>'
-
-
-def render_property_links(prop_name: str, prop_info: dict[str, Any], machine_admin_url: str | None) -> str:
-    links = [
-        str(format_html('<a href="{}">Computed value</a>', get_live_config_url(prop_name))),
-    ]
-    if machine_admin_url:
-        links.append(str(format_html('<a href="{}">Edit override</a>', machine_admin_url)))
-
-    fallback = prop_info.get("x-fallback")
-    if isinstance(fallback, str) and fallback:
-        links.append(str(format_html('<a href="{}">Fallback: <code>{}</code></a>', get_live_config_url(fallback), fallback)))
-
-    aliases = prop_info.get("x-aliases") or []
-    if isinstance(aliases, list):
-        for alias in aliases:
-            if isinstance(alias, str) and alias:
-                links.append(str(format_html('<a href="{}">Alias: <code>{}</code></a>', get_live_config_url(alias), alias)))
-
-    default = prop_info.get("default")
-    if prop_name.endswith("_BINARY") and isinstance(default, str) and default:
-        links.append(str(format_html('<a href="{}">Binary: <code>{}</code></a>', get_environment_binary_url(default), default)))
-
-    return " &nbsp; ".join(links)
-
-
-def render_config_properties_html(properties: dict[str, Any], machine_admin_url: str | None) -> str:
-    header_links = [
-        str(format_html('<a href="{}">Dependencies</a>', ENVIRONMENT_BINARIES_BASE_URL)),
-        str(format_html('<a href="{}">Installed Binaries</a>', INSTALLED_BINARIES_BASE_URL)),
-    ]
-    if machine_admin_url:
-        header_links.insert(0, str(format_html('<a href="{}">Machine Config Editor</a>', machine_admin_url)))
-
-    cards = [
-        f'<div style="margin: 0 0 16px 0;">{" &nbsp; | &nbsp; ".join(header_links)}</div>',
-    ]
-
-    for prop_name, prop_info in properties.items():
-        prop_type = prop_info.get("type", "unknown")
-        if isinstance(prop_type, list):
-            prop_type = " | ".join(str(type_name) for type_name in prop_type)
-        prop_desc = prop_info.get("description", "")
-
-        default_html = ""
-        if "default" in prop_info:
-            default_html = str(
-                format_html(
-                    '<div style="margin-top: 6px;"><b>Default:</b> <code>{}</code></div>',
-                    prop_info["default"],
-                ),
-            )
-
-        description_html = prop_desc or mark_safe('<span style="color: #6e7781;">(no description)</span>')
-        cards.append(
-            str(
-                format_html(
-                    '<div style="margin: 0 0 14px 0; padding: 12px; background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 6px;">'
-                    '<div style="margin-bottom: 6px;">'
-                    '<a href="{}" style="font-weight: 600;"><code>{}</code></a>'
-                    ' <span style="color: #6e7781;">({})</span>'
-                    "</div>"
-                    '<div style="margin-bottom: 6px;">{}</div>'
-                    '<div style="font-size: 0.95em;">{}</div>'
-                    "{}"
-                    "</div>",
-                    get_live_config_url(prop_name),
-                    prop_name,
-                    prop_type,
-                    description_html,
-                    mark_safe(render_property_links(prop_name, prop_info, machine_admin_url)),
-                    mark_safe(default_html),
-                ),
-            ),
-        )
-
-    return "".join(cards)
-
-
-def render_hook_links_html(plugin_name: str, hooks: list[str], source: str) -> str:
-    if not hooks:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    items = []
-    for hook_name in hooks:
-        if source == "builtin":
-            items.append(
-                str(
-                    format_html(
-                        '<div style="margin: 0 0 8px 0;"><a href="{}" target="_blank" rel="noopener noreferrer"><code>{}</code></a></div>',
-                        get_plugin_hook_source_url(plugin_name, hook_name),
-                        hook_name,
-                    ),
-                ),
-            )
-        else:
-            items.append(
-                str(
-                    format_html(
-                        '<div style="margin: 0 0 8px 0;"><code>{}</code></div>',
-                        hook_name,
-                    ),
-                ),
-            )
-    return "".join(items)
-
-
 def render_binary_detail_description(name: str, merged: dict[str, Any], db_binary: Any) -> str:
     installed_binary_url = get_installed_binary_change_url(name, db_binary)
 
@@ -359,59 +116,32 @@ def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]:
 
 
 def get_db_binaries_by_name() -> dict[str, Binary]:
+    """Group Binary rows by a URL-safe canonical name.
+
+    Hooks occasionally emit ``BinaryEvent.name`` carrying an abspath rather
+    than a short binary name (see ``services/binary_service.py``). That used
+    to leak ``name='/Users/.../bin/foo'`` rows into the DB, which then broke
+    ``/admin/environment/binaries`` because the admin URL regex is
+    ``(?P<key>[^/]+)``. Canonicalize at the keying step so duplicates fold
+    into the real binary and the admin link key stays slash-free regardless
+    of legacy DB state.
+    """
+    from archivebox.machine.models import _canonical_binary_name
+
     grouped: dict[str, list[Binary]] = {}
     binary_name_aliases = {
         "youtube-dl": "yt-dlp",
     }
     for binary in Binary.objects.all():
-        canonical_name = binary_name_aliases.get(binary.name, binary.name)
+        canonical_name = _canonical_binary_name(binary.name)
+        canonical_name = binary_name_aliases.get(canonical_name, canonical_name)
+        if not canonical_name:
+            continue
         grouped.setdefault(canonical_name, []).append(binary)
 
     return {name: max(records, key=_binary_sort_key) for name, records in grouped.items()}
 
 
-def get_filesystem_plugins() -> dict[str, dict[str, Any]]:
-    """Discover plugins from filesystem directories."""
-    import json
-    from archivebox.hooks import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR
-
-    plugins = {}
-
-    for base_dir, source in [(BUILTIN_PLUGINS_DIR, "builtin"), (USER_PLUGINS_DIR, "user")]:
-        if not base_dir.exists():
-            continue
-
-        for plugin_dir in base_dir.iterdir():
-            if plugin_dir.is_dir() and not plugin_dir.name.startswith("_"):
-                plugin_id = f"{source}.{plugin_dir.name}"
-
-                # Find hook scripts
-                hooks = []
-                for ext in ("sh", "py", "js"):
-                    hooks.extend(plugin_dir.glob(f"on_*__*.{ext}"))
-
-                # Load config.json if it exists
-                config_file = plugin_dir / "config.json"
-                config_data = None
-                if config_file.exists():
-                    try:
-                        with open(config_file) as f:
-                            config_data = json.load(f)
-                    except (json.JSONDecodeError, OSError):
-                        config_data = None
-
-                plugins[plugin_id] = {
-                    "id": plugin_id,
-                    "name": plugin_dir.name,
-                    "path": str(plugin_dir),
-                    "source": source,
-                    "hooks": [str(h.name) for h in hooks],
-                    "config": config_data,
-                }
-
-    return plugins
-
-
 @render_with_table_view
 def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."
@@ -496,137 +226,13 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     )
 
 
-@render_with_table_view
-def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    assert is_superuser(request), "Must be a superuser to view configuration settings."
-
-    rows = {
-        "Name": [],
-        "Source": [],
-        "Path": [],
-        "Hooks": [],
-        "Config": [],
-    }
-
-    plugins = get_filesystem_plugins()
-
-    for plugin_id, plugin in plugins.items():
-        rows["Name"].append(ItemLink(plugin["name"], key=plugin_id))
-        rows["Source"].append(plugin["source"])
-        rows["Path"].append(format_html("<code>{}</code>", plugin["path"]))
-        rows["Hooks"].append(", ".join(plugin["hooks"]) or "(none)")
-
-        # Show config status
-        if plugin.get("config"):
-            config_properties = plugin["config"].get("properties", {})
-            config_count = len(config_properties)
-            rows["Config"].append(f"✅ {config_count} properties" if config_count > 0 else "✅ present")
-        else:
-            rows["Config"].append("❌ none")
-
-    if not plugins:
-        # Show a helpful message when no plugins found
-        rows["Name"].append("(no plugins found)")
-        rows["Source"].append("-")
-        rows["Path"].append(mark_safe("<code>abx_plugins/plugins/</code> or <code>data/custom_plugins/</code>"))
-        rows["Hooks"].append("-")
-        rows["Config"].append("-")
-
-    return TableContext(
-        title="Installed plugins",
-        table=rows,
-    )
-
-
-@render_with_item_view
-def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-    assert is_superuser(request), "Must be a superuser to view configuration settings."
-
-    plugins = get_filesystem_plugins()
-
-    plugin = plugins.get(key)
-    if not plugin:
-        return ItemContext(
-            slug=key,
-            title=f"Plugin not found: {key}",
-            data=[],
-        )
-
-    # Base fields that all plugins have
-    docs_url = get_plugin_docs_url(plugin["name"])
-    machine_admin_url = get_machine_admin_url()
-    fields = {
-        "id": plugin["id"],
-        "name": plugin["name"],
-        "source": plugin["source"],
-    }
-
-    sections: list[SectionData] = [
-        {
-            "name": plugin["name"],
-            "description": format_html(
-                '<code>{}</code><br/><a href="{}" target="_blank" rel="noopener noreferrer">ABX Plugin Docs</a>',
-                plugin["path"],
-                docs_url,
-            ),
-            "fields": fields,
-            "help_texts": {},
-        },
-    ]
-
-    if plugin["hooks"]:
-        sections.append(
-            {
-                "name": "Hooks",
-                "description": mark_safe(render_hook_links_html(plugin["name"], plugin["hooks"], plugin["source"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-    if plugin.get("config"):
-        sections.append(
-            {
-                "name": "Plugin Metadata",
-                "description": mark_safe(render_plugin_metadata_html(plugin["config"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-        sections.append(
-            {
-                "name": "config.json",
-                "description": mark_safe(render_highlighted_json_block(plugin["config"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-        config_properties = plugin["config"].get("properties", {})
-        if config_properties:
-            sections.append(
-                {
-                    "name": "Config Properties",
-                    "description": mark_safe(render_config_properties_html(config_properties, machine_admin_url)),
-                    "fields": {},
-                    "help_texts": {},
-                },
-            )
-
-    return ItemContext(
-        slug=key,
-        title=plugin["name"],
-        data=sections,
-    )
-
-
 @render_with_table_view
 def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."
 
     rows = {
         "Name": [],
+        "Type": [],
         "State": [],
         "PID": [],
         "Started": [],
@@ -656,12 +262,115 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
             continue
         all_config[config_name] = config_data
 
-    # Add top row for supervisord process manager
+    # Collect every PID we plan to show so we can resolve them to Process rows
+    # in a single query. supervisord's per-worker description carries the pid
+    # in the form ``pid 12345, uptime 0:01:23`` (or just the bare ``pid``
+    # placeholder when stopped); we ignore non-numeric values.
+    process_items = supervisor.getAllProcessInfo()
+    if not isinstance(process_items, list):
+        process_items = []
+
+    def _parse_worker_pid_and_uptime(description: str) -> tuple[int | None, str]:
+        body = description.replace("pid ", "", 1)
+        pid_part, _, uptime_part = body.partition(", uptime ")
+        try:
+            return int(pid_part.strip()), uptime_part.strip()
+        except ValueError:
+            return None, ""
+
+    pids: set[int] = set()
+    supervisor_pid = supervisor.getPID()
+    if isinstance(supervisor_pid, int):
+        pids.add(supervisor_pid)
+    for proc_data in process_items:
+        if not isinstance(proc_data, dict):
+            continue
+        pid_int, _ = _parse_worker_pid_and_uptime(str(proc_data.get("description") or ""))
+        if pid_int is not None:
+            pids.add(pid_int)
+
+    pid_to_process_id: dict[int, str] = {}
+    pid_to_process_type: dict[int, str] = {}
+    if pids:
+        try:
+            from archivebox.machine.models import Machine, Process
+
+            for row in (
+                Process.objects.filter(machine=Machine.current(), pid__in=pids)
+                .order_by("pid", "-started_at", "-created_at")
+                .only("id", "pid", "process_type")
+            ):
+                if row.pid in pid_to_process_id:
+                    continue  # keep the most recent row per PID
+                pid_to_process_id[row.pid] = str(row.id)
+                pid_to_process_type[row.pid] = row.process_type
+
+        except Exception:
+            pass
+
+    def _pid_cell(pid_value: int | None, uptime_str: str = ""):
+        if pid_value is None:
+            return ""
+        pid_text = str(pid_value)
+        process_id = pid_to_process_id.get(pid_value)
+        if process_id:
+            link = format_html('<a href="/admin/machine/process/{}/change/">{}</a>', process_id, pid_text)
+        else:
+            link = format_html("{}", pid_text)
+        if uptime_str:
+            return format_html("{}, uptime {}", link, uptime_str)
+        return link
+
+    # Add top row for supervisord process manager. supervisord exposes its
+    # state + pid over XML-RPC but not its own start time / exit status / uptime,
+    # so we read those from the OS process (or fall back to the Process row
+    # recorded in _record_supervisord_process). Exit status stays blank while
+    # it's RUNNING — supervisord wouldn't be answering RPC if it had exited.
     rows["Name"].append(ItemLink("supervisord", key="supervisord"))
+    rows["Type"].append("supervisord")
     supervisor_state = supervisor.getState()
-    rows["State"].append(str(supervisor_state.get("statename") if isinstance(supervisor_state, dict) else ""))
-    rows["PID"].append(str(supervisor.getPID()))
-    rows["Started"].append("-")
+    state_name = str(supervisor_state.get("statename") if isinstance(supervisor_state, dict) else "")
+    rows["State"].append(state_name)
+
+    supervisor_started = ""
+    supervisor_uptime = ""
+    try:
+        import time as _time
+
+        import psutil
+
+        ps_proc = psutil.Process(supervisor_pid)
+        create_time = ps_proc.create_time()
+        supervisor_started = format_parsed_datetime(create_time)
+        seconds = max(int(_time.time() - create_time), 0)
+        hours, remainder = divmod(seconds, 3600)
+        minutes, secs = divmod(remainder, 60)
+        supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
+    except Exception:
+        try:
+            from archivebox.machine.models import Machine, Process
+
+            row = (
+                Process.objects.filter(
+                    machine=Machine.current(),
+                    process_type=Process.TypeChoices.SUPERVISORD,
+                    pid=supervisor_pid,
+                )
+                .order_by("-started_at")
+                .first()
+            )
+            if row and row.started_at:
+                supervisor_started = row.started_at.strftime("%Y-%m-%d %H:%M:%S")
+                seconds = max(int((timezone.now() - row.started_at).total_seconds()), 0)
+                hours, remainder = divmod(seconds, 3600)
+                minutes, secs = divmod(remainder, 60)
+                supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
+        except Exception:
+            pass
+
+    rows["PID"].append(_pid_cell(supervisor_pid if isinstance(supervisor_pid, int) else None, supervisor_uptime))
+    rows["Started"].append(supervisor_started or "-")
+
     rows["Command"].append("supervisord --configuration=tmp/supervisord.conf")
     rows["Logfile"].append(
         format_html(
@@ -670,12 +379,9 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
             "logs/supervisord.log",
         ),
     )
-    rows["Exit Status"].append("0")
+    rows["Exit Status"].append("" if state_name == "RUNNING" else "-")
 
     # Add a row for each worker process managed by supervisord
-    process_items = supervisor.getAllProcessInfo()
-    if not isinstance(process_items, list):
-        process_items = []
     for proc_data in process_items:
         if not isinstance(proc_data, dict):
             continue
@@ -684,10 +390,15 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
         proc_start = proc_data.get("start")
         proc_logfile = str(proc_data.get("stdout_logfile") or "")
         proc_config = all_config.get(proc_name, {})
+        pid_int, uptime_str = _parse_worker_pid_and_uptime(proc_description)
 
         rows["Name"].append(ItemLink(proc_name, key=proc_name))
+        # Prefer the Process row's process_type when we have one (e.g. "worker",
+        # "hook"); otherwise fall back to the generic "worker" label since
+        # everything in this loop is supervisord-managed.
+        rows["Type"].append(pid_to_process_type.get(pid_int, "worker") if pid_int else "worker")
         rows["State"].append(str(proc_data.get("statename") or ""))
-        rows["PID"].append(proc_description.replace("pid ", ""))
+        rows["PID"].append(_pid_cell(pid_int, uptime_str))
         rows["Started"].append(format_parsed_datetime(proc_start))
         rows["Command"].append(str(proc_config.get("command") or ""))
         rows["Logfile"].append(
@@ -810,7 +521,7 @@ def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
                 f.seek(0)
             last_lines = f.read().decode("utf-8", errors="replace").split("\n")
             non_empty_lines = [line for line in last_lines if line.strip()]
-            rows["Most Recent Lines"].append(non_empty_lines[-1])
+            rows["Most Recent Lines"].append(non_empty_lines[-1] if non_empty_lines else "")
 
     return TableContext(
         title="Debug Log files",

archivebox/core/admin_archiveresults.py

@@ -23,18 +23,18 @@ from archivebox.config import DATA_DIR
 from archivebox.config.common import get_config
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.base_models.admin import BaseModelAdmin
-from archivebox.hooks import get_plugin_icon
-from archivebox.core.host_utils import build_snapshot_url
+from archivebox.plugins.discovery import get_plugin_icon
+from archivebox.plugins.views import LIVE_PLUGIN_BASE_URL
+from archivebox.core.routes_util import build_snapshot_url
 from archivebox.core.widgets import InlineTagEditorWidget
-from archivebox.core.views import LIVE_PLUGIN_BASE_URL
-from archivebox.machine.env_utils import env_to_shell_exports
+from archivebox.machine.env_util import env_to_shell_exports
 
 
 from archivebox.core.models import ArchiveResult, Snapshot
 
 
 def _get_replay_source_url(result: ArchiveResult) -> str:
-    process = getattr(result, "process", None)
+    process = result.process_record
     return str(getattr(process, "url", None) or result.snapshot.url or "")
 
 
@@ -62,7 +62,7 @@ def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str:
 
 
 def get_plugin_admin_url(plugin_name: str) -> str:
-    from archivebox.hooks import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, iter_plugin_dirs
+    from archivebox.plugins.discovery import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, iter_plugin_dirs
 
     plugin_dir = next((path.resolve() for path in iter_plugin_dirs() if path.name == plugin_name), None)
     if plugin_dir:
@@ -105,6 +105,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         "failed": ("#991b1b", "#fee2e2"),  # red
         "queued": ("#6b7280", "#f3f4f6"),  # gray
         "started": ("#92400e", "#fef3c7"),  # amber
+        "paused": ("#1d4ed8", "#dbeafe"),  # blue
         "backoff": ("#92400e", "#fef3c7"),
         "skipped": ("#475569", "#f1f5f9"),
         "noresults": ("#475569", "#f1f5f9"),
@@ -134,24 +135,25 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         # Format timestamp
         end_time = result.end_ts.strftime("%Y-%m-%d %H:%M:%S") if result.end_ts else "-"
 
+        process = result.process_record
         process_display = "-"
-        if result.process_id and result.process:
+        if process:
             process_display = f'''
-                <a href="{reverse("admin:machine_process_change", args=[result.process_id])}"
+                <a href="{reverse("admin:machine_process_change", args=[process.id])}"
                    style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
-                   title="View process">{result.process.pid or "-"}</a>
+                   title="View process">{process.pid or "-"}</a>
             '''
 
         machine_display = "-"
-        if result.process_id and result.process and result.process.machine_id:
+        if process and process.machine_id:
             machine_display = f'''
-                <a href="{reverse("admin:machine_machine_change", args=[result.process.machine_id])}"
+                <a href="{reverse("admin:machine_machine_change", args=[process.machine_id])}"
                    style="color: #2563eb; text-decoration: none; font-size: 12px;"
-                   title="View machine">{result.process.machine.hostname}</a>
+                   title="View machine">{process.machine.hostname}</a>
             '''
 
         # Truncate output for display
-        full_output = result.output_str or "-"
+        full_output = result.output_str_for_display() or "-"
         output_display = full_output[:60]
         if len(full_output) > 60:
             output_display += "..."
@@ -496,7 +498,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
         self.request = request
         request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         saved_list_per_page = self.list_per_page
-        self.list_per_page = min(max(5, request.archivebox_config.SNAPSHOTS_PER_PAGE), 5000)
+        self.list_per_page = request.archivebox_config.SNAPSHOTS_PER_PAGE
         try:
             return super().changelist_view(request, extra_context)
         finally:
@@ -660,20 +662,22 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     @admin.display(description="Process", ordering="process__pid")
     def process_link(self, result):
-        if not result.process_id:
+        process = result.process_record
+        if not process:
             return "-"
-        process_label = result.process.pid if result.process and result.process.pid else "-"
+        process_label = process.pid or "-"
         return format_html(
             '<a href="{}"><code>{}</code></a>',
-            reverse("admin:machine_process_change", args=[result.process_id]),
+            reverse("admin:machine_process_change", args=[process.id]),
             process_label,
         )
 
     @admin.display(description="Machine", ordering="process__machine__hostname")
     def machine_link(self, result):
-        if not result.process_id or not result.process or not result.process.machine_id:
+        process = result.process_record
+        if not process or not process.machine_id:
             return "-"
-        machine = result.process.machine
+        machine = process.machine
         return format_html(
             '<a href="{}"><code>{}</code> {}</a>',
             reverse("admin:machine_machine_change", args=[machine.id]),
@@ -715,12 +719,12 @@ class ArchiveResultAdmin(BaseModelAdmin):
         return format_html(
             '<a href="{}" class="output-link">↗️</a><pre>{}</pre>',
             build_snapshot_url(snapshot_id, output_path, request=request, config=config),
-            result.output_str,
+            result.output_str_for_display(),
         )
 
     @admin.display(description="Output", ordering="output_str")
     def output_str_display(self, result):
-        output_text = str(result.output_str or "").strip()
+        output_text = str(result.output_str_for_display() or "").strip()
         if not output_text:
             return "-"
 
@@ -784,7 +788,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
         snapshot_dir = Path(DATA_DIR) / str(result.pwd).split("data/", 1)[-1]
         output_html = format_html(
             '<pre style="display: inline-block">{}</pre><br/>',
-            result.output_str,
+            result.output_str_for_display(),
         )
         snapshot_id = str(result.snapshot_id)
         request = getattr(self, "request", None)

archivebox/core/admin_snapshots.py

@@ -1,15 +1,17 @@
 __package__ = "archivebox.core"
 
-from functools import lru_cache
 import json
+from functools import lru_cache
+from types import SimpleNamespace
 
 from django.contrib import admin, messages
-from django.urls import path
+from django.urls import path, reverse
 from django.shortcuts import get_object_or_404, redirect
-from django.utils.html import format_html
+from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed
+from django.utils import timezone
+from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
-from django.db.models import Q, Sum, Count, Prefetch
-from django.db.models.functions import Coalesce
+from django.db.models import Q, Count, Exists, OuterRef, Prefetch
 from django import forms
 from django.template import Template, RequestContext
 from django.contrib.admin.helpers import ActionForm
@@ -18,22 +20,32 @@ from archivebox.config.common import get_config
 from archivebox.misc.util import htmldecode, urldecode
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.misc.logging_util import printable_filesize
-from archivebox.search.admin import SearchResultsAdminMixin
-from archivebox.core.host_utils import build_snapshot_url, build_web_url
-from archivebox.hooks import discover_hooks, get_plugin_icon, get_plugin_name, get_plugins
+from archivebox.search.admin import SearchResultsAdminMixin, SearchResultsChangeList
+from archivebox.search.views import admin_snapshot_search_stream_view
+from archivebox.core.routes_util import build_snapshot_url, build_web_url
+from archivebox.core.tag_util import get_or_create_tag
+from archivebox.plugins.hooks import discover_hooks
+from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_plugins
 
 from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
-from archivebox.workers.tasks import bg_archive_snapshots, bg_add
 
 from archivebox.core.models import Tag, Snapshot, ArchiveResult
 from archivebox.core.admin_archiveresults import render_archiveresults_list
+from archivebox.progressmonitor.views import progress_endpoint
+from archivebox.core.permissions import (
+    PERMISSIONS_CHOICES,
+    PERMISSIONS_META,
+    get_snapshot_permissions,
+    normalize_permissions,
+)
 from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget
-from archivebox.crawls.models import Crawl
 
 
 # GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False}
 GLOBAL_CONTEXT = {}
 
+SNAPSHOT_PERMISSION_META = PERMISSIONS_META
+
 
 @lru_cache(maxsize=1)
 def _plugin_sort_order() -> dict[str, int]:
@@ -51,22 +63,12 @@ class SnapshotActionForm(ActionForm):
         )
 
     def clean_tags(self):
-        """Parse comma-separated tag names into Tag objects."""
+        """Parse comma-separated tag names without touching the DB."""
         tags_str = self.cleaned_data.get("tags", "")
         if not tags_str:
             return []
 
-        tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
-        tags = []
-        for name in tag_names:
-            tag, _ = Tag.objects.get_or_create(
-                name__iexact=name,
-                defaults={"name": name},
-            )
-            # Use the existing tag if found by case-insensitive match
-            tag = Tag.objects.filter(name__iexact=name).first() or tag
-            tags.append(tag)
-        return tags
+        return [name.strip() for name in tags_str.split(",") if name.strip()]
 
     # TODO: allow selecting actions for specific extractor plugins? is this useful?
     # plugin = forms.ChoiceField(
@@ -95,6 +97,176 @@ class TagNameListFilter(admin.SimpleListFilter):
         return queryset
 
 
+class SnapshotPermissionsListFilter(admin.SimpleListFilter):
+    title = "permission"
+    parameter_name = "permissions"
+
+    def lookups(self, request, model_admin):
+        return PERMISSIONS_CHOICES
+
+    def queryset(self, request, queryset):
+        value = self.value()
+        if value:
+            return queryset.filter(
+                Q(permissions=value) | (Q(permissions__isnull=True) & Q(crawl__permissions=value)),
+            )
+        return queryset
+
+
+class SnapshotStatusListFilter(admin.SimpleListFilter):
+    title = "snapshot status"
+    parameter_name = "snapshot_status"
+
+    def lookups(self, request, model_admin):
+        return Snapshot.StatusChoices.choices
+
+    def queryset(self, request, queryset):
+        value = self.value()
+        if value in Snapshot.StatusChoices.values:
+            return queryset.filter(status=value)
+        return queryset
+
+
+class SnapshotArchiveStateListFilter(admin.SimpleListFilter):
+    title = "archive state"
+    parameter_name = "archive_state"
+
+    def lookups(self, request, model_admin):
+        return (
+            ("downloaded", "Downloaded"),
+            ("not_downloaded", "Not downloaded"),
+            ("has_output", "Has saved files"),
+            ("empty_output", "No saved files"),
+            ("has_title", "Has title"),
+            ("missing_title", "Missing title"),
+        )
+
+    def queryset(self, request, queryset):
+        value = self.value()
+        if value == "downloaded":
+            return queryset.filter(downloaded_at__isnull=False)
+        if value == "not_downloaded":
+            return queryset.filter(downloaded_at__isnull=True)
+        if value == "has_output":
+            return queryset.filter(output_size__gt=0)
+        if value == "empty_output":
+            return queryset.filter(output_size=0)
+        if value == "has_title":
+            return queryset.exclude(Q(title__isnull=True) | Q(title=""))
+        if value == "missing_title":
+            return queryset.filter(Q(title__isnull=True) | Q(title=""))
+        return queryset
+
+
+class SnapshotSizeListFilter(admin.SimpleListFilter):
+    title = "size"
+    parameter_name = "size"
+
+    def lookups(self, request, model_admin):
+        return (
+            ("1gb", ">1GB"),
+            ("500mb", ">500MB"),
+            ("250mb", ">250MB"),
+            ("100mb", ">100MB"),
+            ("50mb", ">50MB"),
+            ("25mb", ">25MB"),
+        )
+
+    def queryset(self, request, queryset):
+        value = self.value()
+        thresholds = {
+            "1gb": 1024 * 1024 * 1024,
+            "500mb": 500 * 1024 * 1024,
+            "250mb": 250 * 1024 * 1024,
+            "100mb": 100 * 1024 * 1024,
+            "50mb": 50 * 1024 * 1024,
+            "25mb": 25 * 1024 * 1024,
+        }
+        if value in thresholds:
+            return queryset.filter(output_size__gt=thresholds[value])
+        return queryset
+
+
+class SnapshotResultHealthListFilter(admin.SimpleListFilter):
+    title = "ArchiveResult status"
+    parameter_name = "archiveresult_status"
+
+    def lookups(self, request, model_admin):
+        return (
+            ("none", "No ArchiveResults"),
+            ("has_results", "Has ArchiveResults"),
+            ("succeeded", ">50% succeeded"),
+            ("failed", ">50% failed"),
+            ("running", ">50% running"),
+            ("pending", ">50% queued"),
+            ("backoff", ">50% waiting to retry"),
+            ("noresults", ">50% noresults"),
+        )
+
+    def queryset(self, request, queryset):
+        value = self.value()
+        if value:
+            results = ArchiveResult.objects.filter(snapshot_id=OuterRef("pk"))
+            if value == "none":
+                return queryset.annotate(has_results=Exists(results)).filter(has_results=False)
+            if value == "has_results":
+                return queryset.annotate(has_results=Exists(results)).filter(has_results=True)
+            status_by_value = {
+                "succeeded": ArchiveResult.StatusChoices.SUCCEEDED,
+                "failed": ArchiveResult.StatusChoices.FAILED,
+                "running": ArchiveResult.StatusChoices.STARTED,
+                "pending": ArchiveResult.StatusChoices.QUEUED,
+                "backoff": ArchiveResult.StatusChoices.BACKOFF,
+                "noresults": ArchiveResult.StatusChoices.NORESULTS,
+            }
+            if value in status_by_value:
+                # Start from ArchiveResult.status for every majority-status filter.
+                # The (status, snapshot_id) index keeps this plan stable regardless
+                # of which status is most common in a user's collection.
+                return queryset.filter(pk__in=ArchiveResult.snapshot_ids_with_majority_status(status_by_value[value]))
+        return queryset
+
+
+class SnapshotChangeList(SearchResultsChangeList):
+    def __init__(self, request, *args, **kwargs):
+        super().__init__(request, *args, **kwargs)
+        resolver_name = getattr(getattr(request, "resolver_match", None), "url_name", "")
+        self.embedded_changelist = request.GET.get("_embedded") == "crawl"
+        self.snapshot_is_grid_view = not self.embedded_changelist and (
+            resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")
+        )
+
+    def get_results(self, request):
+        super().get_results(request)
+        if request.GET.get("_embedded") == "crawl":
+            self.full_result_count = self.result_count
+        else:
+            self.full_result_count = self.model_admin.get_paginator(
+                request,
+                self.model._default_manager.all().order_by(),
+                self.list_per_page,
+            ).count
+        self.show_full_result_count = True
+
+        snapshot_ids = [obj.pk for obj in self.result_list]
+        if snapshot_ids:
+            results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
+            seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids}
+            rows = (
+                ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
+                .order_by("snapshot_id", "plugin")
+                .values_list("snapshot_id", "plugin", "status", "output_size")
+            )
+            for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000):
+                if plugin in seen_plugins[snapshot_id]:
+                    continue
+                seen_plugins[snapshot_id].add(plugin)
+                results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size))
+
+            for obj in self.result_list:
+                obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
+
+
 class SnapshotAdminForm(forms.ModelForm):
     """Custom form for Snapshot admin with tag editor widget."""
 
@@ -104,6 +276,12 @@ class SnapshotAdminForm(forms.ModelForm):
         widget=TagEditorWidget(),
         help_text="Type tag names and press Enter or Space to add. Click × to remove.",
     )
+    permissions_config = forms.ChoiceField(
+        label="Permissions",
+        choices=PERMISSIONS_CHOICES,
+        required=True,
+        help_text="Per-snapshot visibility. Matching the crawl/persona default clears the per-snapshot override.",
+    )
 
     class Meta:
         model = Snapshot
@@ -116,9 +294,17 @@ class SnapshotAdminForm(forms.ModelForm):
             self.initial["tags_editor"] = ",".join(
                 sorted(tag.name for tag in self.instance.tags.all()),
             )
+            self.initial["permissions_config"] = get_snapshot_permissions(self.instance)
 
     def save(self, commit=True):
         instance = super().save(commit=False)
+        permissions = self.cleaned_data["permissions_config"]
+        config = dict(instance.config or {})
+        if permissions == instance.crawl.permissions:
+            config.pop("PERMISSIONS", None)
+        else:
+            config["PERMISSIONS"] = permissions
+        instance.config = config
 
         # Handle tags_editor field
         if commit:
@@ -150,7 +336,17 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     form = SnapshotAdminForm
     raw_id_fields = ("crawl", "parent_snapshot")
     list_select_related = ()
-    list_display = ("created_at", "preview_icon", "title_str", "tags_inline", "status_with_progress", "files", "size_with_stats")
+    list_display = (
+        "permissions_badge",
+        "created_at",
+        "preview_icon",
+        "title_str",
+        "tags_inline",
+        "status_with_progress",
+        "files",
+        "size_with_stats",
+    )
+    list_display_links = ("created_at",)
     sort_fields = ("title_str", "created_at", "status", "crawl")
     readonly_fields = (
         "admin_actions",
@@ -165,7 +361,17 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         "archiveresults_list",
     )
     search_fields = ("id", "url", "timestamp", "title", "tags__name")
-    list_filter = ("created_at", "downloaded_at", "archiveresult__status", "crawl__created_by", TagNameListFilter)
+    list_filter = (
+        SnapshotPermissionsListFilter,
+        SnapshotStatusListFilter,
+        SnapshotResultHealthListFilter,
+        SnapshotArchiveStateListFilter,
+        SnapshotSizeListFilter,
+        "created_at",
+        "downloaded_at",
+        "crawl__created_by",
+        TagNameListFilter,
+    )
 
     fieldsets = (
         (
@@ -192,7 +398,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         (
             "Tags",
             {
-                "fields": ("tags_editor",),
+                "fields": ("tags_editor", "permissions_config"),
                 "classes": ("card",),
             },
         ),
@@ -242,9 +448,17 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     )
 
     ordering = ["-timestamp"]
-    actions = ["add_tags", "remove_tags", "resnapshot_snapshot", "update_snapshots", "overwrite_snapshots", "delete_snapshots"]
+    actions = [
+        "add_tags",
+        "remove_tags",
+        "resnapshot_snapshot",
+        "update_snapshots",
+        "overwrite_snapshots",
+        "set_snapshot_permissions",
+        "delete_snapshots",
+    ]
     inlines = []  # Removed TagInline, using TagEditorWidget instead
-    list_per_page = 40
+    list_per_page = 50
 
     action_form = SnapshotActionForm
     paginator = AcceleratedPaginator
@@ -252,18 +466,43 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     save_on_top = True
     show_full_result_count = False
 
+    def get_changelist(self, request, **kwargs):
+        return SnapshotChangeList
+
+    def get_ordering(self, request):
+        if request.GET.get("o"):
+            return []
+        return super().get_ordering(request)
+
     def change_view(self, request, object_id, form_url="", extra_context=None):
         request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         extra_context = extra_context or {}
         extra_context["CONFIG"] = request.archivebox_config
+        snapshot = self.get_object(request, object_id)
+        if snapshot and snapshot.status in {
+            Snapshot.StatusChoices.QUEUED,
+            Snapshot.StatusChoices.STARTED,
+            Snapshot.StatusChoices.PAUSED,
+        }:
+            extra_context["progress_auto_expand"] = True
+            extra_context["progress_endpoint"] = progress_endpoint("snapshot", snapshot.id)
         return super().change_view(request, object_id, form_url, extra_context | GLOBAL_CONTEXT)
 
     def changelist_view(self, request, extra_context=None):
         self.request = request
         request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         saved_list_per_page = self.list_per_page
-        self.list_per_page = min(max(5, request.archivebox_config.SNAPSHOTS_PER_PAGE), 25)
+        embedded_changelist = request.GET.get("_embedded") == "crawl"
+        if embedded_changelist:
+            try:
+                requested_per_page = int(request.GET.get("per_page", "200"))
+            except ValueError:
+                requested_per_page = 200
+            self.list_per_page = min(max(200, requested_per_page), 500)
+        else:
+            self.list_per_page = request.archivebox_config.SNAPSHOTS_PER_PAGE
         extra_context = extra_context or {}
+        extra_context["embedded_changelist"] = embedded_changelist
         extra_context["CONFIG"] = request.archivebox_config
         try:
             try:
@@ -281,6 +520,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         actions.pop("delete_selected", None)
         return actions
 
+    def lookup_allowed(self, lookup, value, request=None):
+        if lookup in {"crawl__id__exact", "crawl_id__exact", "crawl_id"}:
+            return True
+        return super().lookup_allowed(lookup, value, request=request)
+
     def get_snapshot_view_url(self, obj: Snapshot) -> str:
         request = getattr(self, "request", None)
         return build_snapshot_url(str(obj.id), request=request, config=getattr(request, "archivebox_config", None))
@@ -296,10 +540,79 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         urls = super().get_urls()
         custom_urls = [
             path("grid/", self.admin_site.admin_view(self.grid_view), name="grid"),
+            path("search-stream/", self.admin_site.admin_view(self.search_stream_view), name="core_snapshot_search_stream"),
             path("<path:object_id>/redo-failed/", self.admin_site.admin_view(self.redo_failed_view), name="core_snapshot_redo_failed"),
+            path(
+                "<path:object_id>/set-permissions/",
+                self.admin_site.admin_view(self.set_permissions_view),
+                name="core_snapshot_set_permissions",
+            ),
         ]
         return custom_urls + urls
 
+    def search_stream_view(self, request):
+        return admin_snapshot_search_stream_view(self, request)
+
+    def set_permissions_view(self, request, object_id):
+        if request.method != "POST":
+            return HttpResponseNotAllowed(["POST"])
+
+        permissions = (request.POST.get("permissions") or "").strip().lower()
+        if permissions not in dict(PERMISSIONS_CHOICES):
+            return HttpResponseBadRequest("Invalid permissions value")
+
+        snapshot = get_object_or_404(Snapshot.objects.select_related("crawl"), pk=object_id)
+        config = dict(snapshot.config or {})
+        if permissions == snapshot.crawl.permissions:
+            config.pop("PERMISSIONS", None)
+        else:
+            config["PERMISSIONS"] = permissions
+
+        # Keep the quick-edit write to one targeted UPDATE so SQLite only holds
+        # the write lock for the permission/config change itself. safe_update()
+        # keeps this from overwriting a concurrent admin/runner edit to the
+        # same Snapshot after the row was loaded above.
+        snapshot.safe_update({"config": config, "modified_at": timezone.now()}, refresh=False)
+        icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
+        return JsonResponse({"permissions": permissions, "icon": icon, "label": label, "fg": fg, "bg": bg})
+
+    @admin.action(description="Permissions ▾")
+    def set_snapshot_permissions(self, request, queryset):
+        permissions = (request.POST.get("permissions") or "").strip().lower()
+        if permissions not in dict(PERMISSIONS_CHOICES):
+            messages.error(request, "Choose a valid permissions value.")
+            return
+        updated = self.update_snapshot_permissions(queryset, permissions)
+        messages.success(request, f"Set permissions to {permissions} on {updated} snapshot(s).")
+
+    def update_snapshot_permissions(self, queryset, permissions):
+        now = timezone.now()
+        updated = 0
+        batch = []
+        snapshots = (
+            queryset.select_related(None)
+            .select_related("crawl")
+            .only("id", "config", "crawl__id", "crawl__permissions")
+            .prefetch_related(None)
+        )
+        for snapshot in snapshots.iterator(chunk_size=500):
+            config = dict(snapshot.config or {})
+            if permissions == snapshot.crawl.permissions:
+                config.pop("PERMISSIONS", None)
+            else:
+                config["PERMISSIONS"] = permissions
+            snapshot.config = config
+            snapshot.modified_at = now
+            batch.append(snapshot)
+            if len(batch) >= 500:
+                Snapshot.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
+                updated += len(batch)
+                batch.clear()
+        if batch:
+            Snapshot.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
+            updated += len(batch)
+        return updated
+
     def redo_failed_view(self, request, object_id):
         snapshot = get_object_or_404(Snapshot, pk=object_id)
 
@@ -318,63 +631,110 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
         return redirect(snapshot.admin_change_url)
 
-    # def get_queryset(self, request):
-    #     # tags_qs = SnapshotTag.objects.all().select_related('tag')
-    #     # prefetch = Prefetch('snapshottag_set', queryset=tags_qs)
-
-    #     self.request = request
-    #     return super().get_queryset(request).prefetch_related('archiveresult_set').distinct()  # .annotate(archiveresult_count=Count('archiveresult'))
     def get_queryset(self, request):
         self.request = request
         ordering_fields = self._get_ordering_fields(request)
-        needs_size_sort = "size_with_stats" in ordering_fields
         needs_files_sort = "files" in ordering_fields
         needs_tags_sort = "tags_inline" in ordering_fields
         is_change_view = getattr(getattr(request, "resolver_match", None), "url_name", "") == "core_snapshot_change"
-
-        prefetch_qs = ArchiveResult.objects.only(
-            "id",
-            "snapshot_id",
-            "plugin",
-            "status",
-            "output_size",
-            "output_files",
-            "output_str",
-        )
-        if not is_change_view:
-            prefetch_qs = prefetch_qs.filter(Q(status="succeeded"))
-
-        qs = (
-            super()
-            .get_queryset(request)
-            .defer("config", "notes")
-            .prefetch_related(
-                Prefetch("crawl", queryset=Crawl.objects.select_related("created_by")),
-                "tags",
-                Prefetch("archiveresult_set", queryset=prefetch_qs),
-            )
-        )
-
-        if needs_size_sort:
-            qs = qs.annotate(
-                output_size_sum=Coalesce(
-                    Sum("archiveresult__output_size"),
-                    0,
+        prefetches = ["tags"]
+        if is_change_view:
+            prefetches.append(
+                Prefetch(
+                    "archiveresult_set",
+                    queryset=ArchiveResult.objects.only(
+                        "id",
+                        "snapshot_id",
+                        "plugin",
+                        "status",
+                        "output_size",
+                    ),
                 ),
             )
 
+        qs = super().get_queryset(request).select_related("crawl__created_by")
+        if is_change_view:
+            qs = qs.defer("notes")
+        else:
+            qs = qs.only(
+                "id",
+                "created_at",
+                "url",
+                "timestamp",
+                "bookmarked_at",
+                "crawl_id",
+                "title",
+                "status",
+                "fs_version",
+                "output_size",
+                "permissions",
+                "crawl__id",
+                "crawl__permissions",
+                "crawl__persona_id",
+                "crawl__status",
+                "crawl__created_by_id",
+                "crawl__created_by__id",
+                "crawl__created_by__username",
+            )
+        qs = qs.prefetch_related(*prefetches)
         if needs_files_sort:
             qs = qs.annotate(
-                ar_succeeded_count=Count(
-                    "archiveresult",
-                    filter=Q(archiveresult__status="succeeded"),
-                ),
+                ar_succeeded_count=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
             )
         if needs_tags_sort:
             qs = qs.annotate(tag_count=Count("tags", distinct=True))
 
         return qs
 
+    @admin.display(description="👁", ordering="permissions")
+    def permissions_badge(self, obj):
+        permissions = getattr(obj, "snapshot_permissions", None)
+        if permissions is None:
+            if obj.permissions:
+                permissions = obj.permissions
+            else:
+                permissions = obj.crawl.permissions
+        permissions = normalize_permissions(permissions)
+        icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
+        menu_items = format_html_join(
+            "",
+            (
+                '<button type="button" class="snapshot-permissions-menu-item{}" data-permissions="{}">'
+                '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
+                "<span>{}</span>"
+                "</button>"
+            ),
+            (
+                (
+                    " is-active" if choice_value == permissions else "",
+                    choice_value,
+                    choice_fg,
+                    choice_bg,
+                    choice_icon,
+                    choice_label,
+                )
+                for choice_value, choice_label in PERMISSIONS_CHOICES
+                for choice_icon, _choice_title, choice_fg, choice_bg in [SNAPSHOT_PERMISSION_META[choice_value]]
+            ),
+        )
+        return format_html(
+            '<span class="snapshot-permissions-quick" data-current-permissions="{}" data-permissions-url="{}">'
+            '<button type="button" class="snapshot-permissions-button snapshot-permissions-{}" title="{}" aria-label="Change snapshot permissions: {}" aria-expanded="false">'
+            '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
+            "</button>"
+            '<span class="snapshot-permissions-menu" role="menu" hidden>{}</span>'
+            "</span>",
+            permissions,
+            reverse(f"{self.admin_site.name}:core_snapshot_set_permissions", args=[obj.pk]),
+            permissions,
+            label,
+            label,
+            fg,
+            bg,
+            icon,
+            menu_items,
+        )
+
     @admin.display(description="Imported Timestamp")
     def imported_timestamp(self, obj):
         context = RequestContext(
@@ -555,15 +915,15 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Tags", ordering="tag_count")
     def tags_inline(self, obj):
-        widget = InlineTagEditorWidget(snapshot_id=str(obj.pk))
+        widget = InlineTagEditorWidget(snapshot_id=str(obj.pk), editable=True)
         tags = self._get_prefetched_tags(obj)
         tags_html = widget.render(
-            name=f"tags_{obj.pk}",
+            name=f"tags_inline_{obj.pk}",
             value=tags if tags is not None else obj.tags.all(),
-            attrs={"id": f"tags_{obj.pk}"},
+            attrs={"id": f"tags_inline_{obj.pk}"},
             snapshot_id=str(obj.pk),
         )
-        return mark_safe(f'<span class="tags-inline-editor">{tags_html}</span>')
+        return mark_safe(f'<span class="tags-inline-editor tags-inline-editor--compact">{tags_html}</span>')
 
     @admin.display(description="Tags")
     def tags_badges(self, obj):
@@ -716,13 +1076,13 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def files(self, obj):
         results = self._get_prefetched_results(obj)
         if results is None:
-            results = obj.archiveresult_set.only("plugin", "status", "output_files", "output_str")
+            results = obj.archiveresult_set.only("plugin", "status", "output_size")
 
         plugins_with_output: dict[str, ArchiveResult] = {}
         for result in results:
             if result.status != ArchiveResult.StatusChoices.SUCCEEDED:
                 continue
-            if not (result.output_files or str(result.output_str or "").strip()):
+            if not result.output_size:
                 continue
             plugins_with_output.setdefault(result.plugin, result)
 
@@ -734,15 +1094,21 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             key=lambda result: (_plugin_sort_order().get(result.plugin, 9999), result.plugin),
         )
         visible_results = sorted_results[:14]
-        output = [
-            format_html(
-                '<a href="{}" class="exists-True" title="{}">{}</a>',
-                f"/{obj.archive_path_from_db}/{result.plugin}/",
-                result.plugin,
-                get_plugin_icon(result.plugin),
+        output = []
+        request = getattr(self, "request", None)
+        config = getattr(request, "archivebox_config", None)
+        for result in visible_results:
+            icon = mark_safe(get_plugin_icon(result.plugin))
+            if not icon.strip():
+                continue
+            output.append(
+                format_html(
+                    '<a href="{}" class="exists-True" title="{}">{}</a>',
+                    build_web_url(f"/{obj.archive_path_from_db}/{result.plugin}/", request=request, config=config),
+                    result.plugin,
+                    icon,
+                ),
             )
-            for result in visible_results
-        ]
         if len(sorted_results) > len(visible_results):
             output.append(
                 format_html(
@@ -788,6 +1154,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         status_colors = {
             "queued": ("#f59e0b", "#fef3c7"),  # amber
             "started": ("#3b82f6", "#dbeafe"),  # blue
+            "paused": ("#1d4ed8", "#dbeafe"),  # blue
             "sealed": ("#10b981", "#d1fae5"),  # green
             "succeeded": ("#10b981", "#d1fae5"),  # green
             "failed": ("#ef4444", "#fee2e2"),  # red
@@ -840,7 +1207,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(
         description="Size",
-        ordering="output_size_sum",
+        ordering="output_size",
     )
     def size_with_stats(self, obj):
         """Show archive size with output size from archive results."""
@@ -902,14 +1269,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         pending = max(total - succeeded - failed - running - skipped - noresults, 0)
         completed = succeeded + failed + skipped + noresults
         percent = int((completed / total * 100) if total > 0 else 0)
-        is_sealed = obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED)
-        output_size = None
-
-        if hasattr(obj, "output_size_sum"):
-            output_size = obj.output_size_sum or 0
-        else:
-            output_size = sum(r.output_size or 0 for r in results)
-
+        is_sealed = obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED, obj.StatusChoices.PAUSED)
         stats = {
             "total": total,
             "succeeded": succeeded,
@@ -919,13 +1279,15 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             "skipped": skipped,
             "noresults": noresults,
             "percent": percent,
-            "output_size": output_size or 0,
+            "output_size": obj.output_size or 0,
             "is_sealed": is_sealed,
         }
         obj._admin_progress_stats = stats
         return stats
 
     def _get_prefetched_results(self, obj):
+        if "_admin_archiveresults" in obj.__dict__:
+            return obj.__dict__["_admin_archiveresults"]
         if hasattr(obj, "_prefetched_objects_cache") and "archiveresult_set" in obj._prefetched_objects_cache:
             return obj.archiveresult_set.all()
         return None
@@ -939,8 +1301,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             crawl = getattr(obj, "crawl", None)
             snapshot_config = getattr(obj, "config", None) or {}
             crawl_config = getattr(crawl, "config", None) or {}
-            crawl_persona_id = getattr(crawl, "persona_id", None)
-            has_scoped_config = bool(snapshot_config or crawl_config or crawl_persona_id)
+            has_scoped_config = bool(snapshot_config or crawl_config)
 
             if request is not None and not has_scoped_config:
                 cached_total = getattr(request, "archivebox_expected_snapshot_hook_total", None)
@@ -958,7 +1319,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                 if snapshot_config:
                     cache_key = ("snapshot", json.dumps(snapshot_config, sort_keys=True, default=str))
                 else:
-                    cache_key = ("crawl", json.dumps(crawl_config, sort_keys=True, default=str), crawl_persona_id)
+                    cache_key = ("crawl", json.dumps(crawl_config, sort_keys=True, default=str))
                 cached_total = scoped_cache.get(cache_key)
                 if cached_total is None:
                     config = get_config(crawl=crawl, snapshot=obj if snapshot_config else None)
@@ -1009,31 +1370,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return format_html('<span style="color: {};">{}</span>', color, h)
 
     def grid_view(self, request, extra_context=None):
-
-        # cl = self.get_changelist_instance(request)
-
-        # Save before monkey patching to restore for changelist list view
-        admin_cls = type(self)
-        saved_change_list_template = admin_cls.change_list_template
-        saved_list_per_page = admin_cls.list_per_page
-        saved_list_max_show_all = admin_cls.list_max_show_all
-
-        # Monkey patch here plus core_tags.py
-        admin_cls.change_list_template = "private_index_grid.html"
-        config = getattr(request, "archivebox_config", None) or get_config()
-        request.archivebox_config = config
-        admin_cls.list_per_page = config.SNAPSHOTS_PER_PAGE
-        admin_cls.list_max_show_all = admin_cls.list_per_page
-
-        # Call monkey patched view
-        rendered_response = self.changelist_view(request, extra_context=extra_context)
-
-        # Restore values
-        admin_cls.change_list_template = saved_change_list_template
-        admin_cls.list_per_page = saved_list_per_page
-        admin_cls.list_max_show_all = saved_list_max_show_all
-
-        return rendered_response
+        extra_context = extra_context or {}
+        extra_context["snapshot_is_grid_view"] = True
+        return self.changelist_view(request, extra_context=extra_context)
 
     # for debugging, uncomment this to print all requests:
     # def changelist_view(self, request, extra_context=None):
@@ -1070,18 +1409,30 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             messages.info(request, "No valid snapshot URLs were found to archive.")
             return
 
-        bg_add({"urls": urls})
+        from archivebox.cli.archivebox_add import add
+
+        # "Archive Now" is an explicit user re-archive — force ONLY_NEW=False
+        # on the resulting crawl so existing snapshots don't cause the crawl to
+        # seal immediately with zero new snapshots (the default ONLY_NEW=True
+        # would skip any URLs that have ever been archived before).
+        crawl, _ = add(urls=urls, bg=True, config={"ONLY_NEW": False})
 
         messages.success(
             request,
-            f"Creating 1 new crawl with {len(snapshots)} fresh snapshots. The background runner will process them.",
+            f"Created 1 queued crawl with {len(snapshots)} URL(s). The background runner will create snapshots and process them.",
         )
 
+        # Redirect to the new crawl's admin page so the user lands on the
+        # work-in-progress crawl, not the old snapshot they re-archived from.
+        # A snapshot-view redirect would race the runner — the new snapshot
+        # may sit queued for a while before the runner creates the DB row.
+        return redirect(f"/admin/crawls/crawl/{crawl.id}/change/#snapshots")
+
     @admin.action(
         description="🔄 Redo",
     )
     def overwrite_snapshots(self, request, queryset):
-        queued = bg_archive_snapshots(queryset, kwargs={"overwrite": True})
+        queued = sum(snapshot.archive(overwrite=True) for snapshot in queryset)
 
         messages.success(
             request,
@@ -1123,28 +1474,24 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             messages.warning(request, "No tags specified.")
             return
 
-        # Parse comma-separated tag names and get/create Tag objects
         tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
         tags = []
         for name in tag_names:
-            tag, _ = Tag.objects.get_or_create(
-                name__iexact=name,
-                defaults={"name": name},
+            tag, _ = get_or_create_tag(
+                name,
+                created_by=request.user if request.user.is_authenticated else None,
             )
-            tag = Tag.objects.filter(name__iexact=name).first() or tag
             tags.append(tag)
 
         # Get snapshot IDs efficiently (works with select_across for all pages)
         snapshot_ids = list(queryset.values_list("id", flat=True))
         num_snapshots = len(snapshot_ids)
 
-        print("[+] Adding tags", [t.name for t in tags], "to", num_snapshots, "Snapshots")
-
-        # Bulk create M2M relationships (1 query per tag, not per snapshot)
         for tag in tags:
             SnapshotTag.objects.bulk_create(
-                [SnapshotTag(snapshot_id=sid, tag=tag) for sid in snapshot_ids],
-                ignore_conflicts=True,  # Skip if relationship already exists
+                [SnapshotTag(snapshot_id=sid, tag_id=tag.pk) for sid in snapshot_ids],
+                ignore_conflicts=True,
+                batch_size=1000,
             )
 
         messages.success(
@@ -1181,9 +1528,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         num_snapshots = len(snapshot_ids)
         tag_ids = [t.pk for t in tags]
 
-        print("[-] Removing tags", [t.name for t in tags], "from", num_snapshots, "Snapshots")
-
-        # Bulk delete M2M relationships (1 query total, not per snapshot)
         deleted_count, _ = SnapshotTag.objects.filter(
             snapshot_id__in=snapshot_ids,
             tag_id__in=tag_ids,

archivebox/core/admin_tags.py

@@ -13,7 +13,7 @@ from django.utils.safestring import mark_safe
 
 from archivebox.base_models.admin import BaseModelAdmin
 from archivebox.core.models import SnapshotTag, Tag
-from archivebox.core.tag_utils import (
+from archivebox.core.tag_util import (
     TAG_HAS_SNAPSHOTS_CHOICES,
     TAG_SORT_CHOICES,
     build_tag_cards,
@@ -24,7 +24,7 @@ from archivebox.core.tag_utils import (
     normalize_has_snapshots_filter,
     normalize_tag_sort,
 )
-from archivebox.core.host_utils import build_snapshot_url
+from archivebox.core.routes_util import build_snapshot_url
 
 
 class TagInline(admin.TabularInline):

archivebox/core/apps.py

@@ -16,24 +16,15 @@ class CoreConfig(AppConfig):
         from archivebox.core.admin_site import register_admin_site
 
         register_admin_site()
+        from archivebox.base_models.models import ModelWithOutputDir
+
+        ModelWithOutputDir.register_delete_signal()
 
         # Import models to register state machines with the registry
         # Skip during makemigrations to avoid premature state machine access
         if "makemigrations" not in sys.argv:
             from archivebox.core import models  # noqa: F401
 
-        pidfile = os.environ.get("ARCHIVEBOX_RUNSERVER_PIDFILE")
-        if pidfile:
-            should_write_pid = True
-            if os.environ.get("ARCHIVEBOX_AUTORELOAD") == "1":
-                should_write_pid = os.environ.get(DJANGO_AUTORELOAD_ENV) == "true"
-            if should_write_pid:
-                try:
-                    with open(pidfile, "w") as handle:
-                        handle.write(str(os.getpid()))
-                except Exception:
-                    pass
-
         def _should_prepare_runtime() -> bool:
             if os.environ.get("ARCHIVEBOX_RUNSERVER") == "1":
                 if os.environ.get("ARCHIVEBOX_AUTORELOAD") == "1":
@@ -42,8 +33,13 @@ class CoreConfig(AppConfig):
             return False
 
         if _should_prepare_runtime():
-            from archivebox.machine.models import Process, Machine
+            from archivebox.config import CONSTANTS
+            from archivebox.machine.models import Process
 
-            Process.cleanup_stale_running()
-            Process.cleanup_orphaned_workers()
-            Machine.current()
+            Process.current().mark_running(
+                process_type=Process.TypeChoices.WORKER,
+                worker_type="worker_runserver",
+                pwd=str(CONSTANTS.DATA_DIR),
+                url=os.environ.get("ARCHIVEBOX_RUNSERVER_BIND_URL") or "",
+                timeout=CONSTANTS.MAX_HOOK_RUNTIME_SECONDS,
+            )

archivebox/core/asgi.py

@@ -12,6 +12,65 @@ from django.core.asgi import get_asgi_application
 
 setup_django(in_memory_db=False, check_db=True)
 
+
+def _patch_thread_sensitive_context_shutdown() -> None:
+    """Stop ``ThreadSensitiveContext.__aexit__`` from blocking the daphne loop.
+
+    Django 6.0's ASGIHandler wraps every request in ``async with
+    ThreadSensitiveContext():`` (django/core/handlers/asgi.py:169). On exit
+    asgiref calls ``executor.shutdown()`` with the default ``wait=True``
+    (asgiref/sync.py:148), which is a *synchronous* ``Thread.join()`` inside
+    an async function — so it blocks the daphne event loop until the
+    executor's worker thread exits.
+
+    That's normally fine because the request handler has already awaited
+    every ``sync_to_async`` it submitted, so the worker is idle and dies as
+    soon as the shutdown sentinel reaches it. The blocking turns into a
+    problem when a client disconnects mid-request:
+
+    * ``SyncToAsync.__call__`` shields the executor work with
+      ``await asyncio.shield(exec_coro)`` (asgiref/sync.py:506) so that the
+      sync DB call doesn't get torn down halfway through.
+    * On cancellation it calls ``exec_coro.cancel()`` (line 522) which only
+      flips the asyncio ``Future`` to cancelled — the underlying thread
+      keeps running the SQL query.
+    * Control unwinds to ``__aexit__`` while the orphaned thread is still
+      mid-query. ``shutdown(wait=True)`` then blocks the event loop until
+      that orphan finishes.
+
+    Under heavy SQLite contention (the load-test scenario that surfaced
+    this on cabbage) those orphan threads can take 30 seconds each waiting
+    for write locks, and the daphne loop is single-threaded — so every
+    such orphan stalls every other in-flight request, healthchecks time
+    out, and the container goes ``unhealthy``.
+
+    Switching to ``shutdown(wait=False)`` queues the sentinel and returns
+    immediately; the worker thread still exits cleanly once its current
+    task finishes, and asgiref's ``WeakKeyDictionary`` releases the
+    executor as soon as the request's context is GC'd. No per-request
+    teardown guarantee is lost — there was no caller relying on it.
+    """
+    from asgiref import sync as _asgiref_sync
+
+    original_aexit = _asgiref_sync.ThreadSensitiveContext.__aexit__
+
+    async def __aexit__(self, exc, value, tb):  # type: ignore[no-redef]
+        if not self.token:
+            return
+        executor = _asgiref_sync.SyncToAsync.context_to_thread_executor.pop(self, None)
+        if executor is not None:
+            executor.shutdown(wait=False)
+        _asgiref_sync.SyncToAsync.thread_sensitive_context.reset(self.token)
+
+    # Idempotent: only patch once even if asgi.py is reloaded.
+    if getattr(original_aexit, "_archivebox_patched", False):
+        return
+    __aexit__._archivebox_patched = True  # type: ignore[attr-defined]
+    _asgiref_sync.ThreadSensitiveContext.__aexit__ = __aexit__
+
+
+_patch_thread_sensitive_context_shutdown()
+
 # Standard Django ASGI application (no websockets/channels needed)
 application = get_asgi_application()
 

archivebox/core/context_processors.py

@@ -0,0 +1,9 @@
+from archivebox.config import VERSION
+from archivebox.config.version import get_COMMIT_HASH
+
+
+def archivebox_globals(request):
+    return {
+        "VERSION": VERSION,
+        "STATIC_CACHE_KEY": (get_COMMIT_HASH() or VERSION or "dev").strip(),
+    }

archivebox/core/forms.py

@@ -1,20 +1,24 @@
 __package__ = "archivebox.core"
 
-import json
-from collections.abc import Iterable, Mapping
-from pathlib import Path
-from typing import Any
+import re
+from decimal import Decimal, InvalidOperation, ROUND_CEILING
 
 from django import forms
-from django.utils.html import format_html
 
 from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
 from taggit.utils import edit_string_for_tags, parse_tags
 from archivebox.base_models.admin import KeyValueWidget
-from archivebox.crawls.schedule_utils import validate_schedule
-from archivebox.config.common import get_config
+from archivebox.crawls.schedule_util import validate_schedule
+from archivebox.config.common import get_config, parse_delete_after
+from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions, is_admin_user
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
-from archivebox.hooks import get_plugins, discover_plugin_configs, get_plugin_icon
+from archivebox.plugins.discovery import get_plugins
+from archivebox.plugins.forms import (
+    PLUGIN_GROUP_DEFINITIONS,
+    TIMEOUT_INPUT_PATTERN,
+    PluginConfigFormMixin,
+    get_choice_field,
+)
 from archivebox.personas.models import Persona
 
 DEPTH_CHOICES = (
@@ -25,481 +29,6 @@ DEPTH_CHOICES = (
     ("4", "depth = 4 (+ URLs four hops away)"),
 )
 
-PLUGIN_CONFIG_FIELD_PREFIX = "plugin_config__"
-PLUGIN_GROUP_DEFINITIONS = (
-    (
-        "main_plugins",
-        "Main",
-        "",
-        "",
-        "",
-        (
-            "dom",
-            "screenshot",
-            "pdf",
-            "singlefile",
-            "wget",
-            "archivedotorg",
-            "chrome_mhtml",
-        ),
-    ),
-    (
-        "page_setup_plugins",
-        "Page Setup",
-        "",
-        "",
-        "",
-        (
-            "chrome",
-            "infiniscroll",
-            "modalcloser",
-            "ublock",
-            "istilldontcareaboutcookies",
-            "twocaptcha",
-            "claudechrome",
-        ),
-    ),
-    (
-        "media_plugins",
-        "Media",
-        "",
-        "",
-        "",
-        (
-            "staticfile",
-            "responses",
-            "ytdlp",
-            "gallerydl",
-            "git",
-        ),
-    ),
-    (
-        "text_plugins",
-        "Text",
-        "",
-        "",
-        "",
-        (
-            "readability",
-            "htmltotext",
-            "defuddle",
-            "forumdl",
-            "mercury",
-            "trafilatura",
-            "liteparse",
-            "opendataloader",
-            "papersdl",
-        ),
-    ),
-    (
-        "metadata_plugins",
-        "Metadata",
-        "",
-        "",
-        "",
-        (
-            "title",
-            "favicon",
-            "headers",
-            "redirects",
-            "accessibility",
-            "consolelog",
-            "sslcerts",
-            "dns",
-            "seo",
-            "hashes",
-        ),
-    ),
-    (
-        "postprocessing_plugins",
-        "Postprocessing",
-        "",
-        "",
-        "",
-        (
-            "parse_dom_outlinks",
-            "parse_html_urls",
-            "parse_jsonl_urls",
-            "parse_netscape_urls",
-            "parse_rss_urls",
-            "parse_txt_urls",
-            "claudecode",
-            "claudecodecleanup",
-            "claudecodeextract",
-        ),
-    ),
-)
-HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
-    "apt",
-    "base",
-    "bash",
-    "brew",
-    "cargo",
-    "chromewebstore",
-    "env",
-    "media",
-    "npm",
-    "pip",
-    "puppeteer",
-    "search_backend_ripgrep",
-    "search_backend_sonic",
-    "search_backend_sqlite",
-    "ssl",
-}
-
-
-def get_plugin_choices():
-    """Get available extractor plugins from discovered hooks."""
-    return [(name, name) for name in get_plugins()]
-
-
-def get_plugin_choice_label(plugin_name: str, plugin_configs: dict[str, dict]) -> str:
-    schema = plugin_configs.get(plugin_name, {})
-    description = str(schema.get("description") or "").strip()
-    if not description:
-        return plugin_name
-    icon_html = get_plugin_icon(plugin_name)
-
-    return format_html(
-        '<span class="plugin-choice-icon">{}</span><span class="plugin-choice-name">{}</span><a class="plugin-choice-description" href="https://archivebox.github.io/abx-plugins/#{}" target="_blank" rel="noopener noreferrer">{}</a>',
-        icon_html,
-        plugin_name,
-        plugin_name,
-        description,
-    )
-
-
-def get_choice_field(form: forms.Form, name: str) -> forms.ChoiceField:
-    field = form.fields[name]
-    if not isinstance(field, forms.ChoiceField):
-        raise TypeError(f"{name} must be a ChoiceField")
-    return field
-
-
-def _plugin_config_input_name(plugin_name: str, config_key: str) -> str:
-    return f"{PLUGIN_CONFIG_FIELD_PREFIX}{plugin_name}__{config_key}"
-
-
-def _schema_types(schema: Mapping[str, Any]) -> list[str]:
-    raw_type = schema.get("type") or "string"
-    if isinstance(raw_type, list):
-        return [str(item) for item in raw_type]
-    return [str(raw_type)]
-
-
-def _jsonish(value: Any) -> str:
-    if isinstance(value, str):
-        return value
-    return json.dumps(value, sort_keys=True, default=str)
-
-
-def _same_config_value(left: Any, right: Any) -> bool:
-    return json.dumps(left, sort_keys=True, default=str) == json.dumps(right, sort_keys=True, default=str)
-
-
-def _coerce_plugin_config_value(raw_value: Any, schema: Mapping[str, Any]) -> Any:
-    schema_types = _schema_types(schema)
-
-    if "boolean" in schema_types:
-        if isinstance(raw_value, bool):
-            return raw_value
-        value = str(raw_value).strip().lower()
-        if value in {"true", "1", "yes", "on"}:
-            return True
-        if value in {"false", "0", "no", "off", ""}:
-            return False
-        raise forms.ValidationError("Must be true or false.")
-
-    if "integer" in schema_types:
-        value = int(str(raw_value).strip())
-        minimum = schema.get("minimum")
-        maximum = schema.get("maximum")
-        if minimum is not None and value < int(minimum):
-            raise forms.ValidationError(f"Must be at least {minimum}.")
-        if maximum is not None and value > int(maximum):
-            raise forms.ValidationError(f"Must be at most {maximum}.")
-        return value
-
-    if "number" in schema_types:
-        value = float(str(raw_value).strip())
-        minimum = schema.get("minimum")
-        maximum = schema.get("maximum")
-        if minimum is not None and value < float(minimum):
-            raise forms.ValidationError(f"Must be at least {minimum}.")
-        if maximum is not None and value > float(maximum):
-            raise forms.ValidationError(f"Must be at most {maximum}.")
-        return value
-
-    if "array" in schema_types:
-        if isinstance(raw_value, list):
-            return raw_value
-        value = str(raw_value).strip()
-        if not value:
-            return []
-        if value.startswith("["):
-            parsed = json.loads(value)
-            if not isinstance(parsed, list):
-                raise forms.ValidationError("Must be a JSON array.")
-            return parsed
-        return [item.strip() for item in value.replace(",", "\n").splitlines() if item.strip()]
-
-    if "object" in schema_types:
-        value = str(raw_value).strip()
-        if not value:
-            return {}
-        parsed = json.loads(value)
-        if not isinstance(parsed, dict):
-            raise forms.ValidationError("Must be a JSON object.")
-        return parsed
-
-    value = str(raw_value)
-    enum = schema.get("enum")
-    if isinstance(enum, list) and enum and value not in {str(item) for item in enum}:
-        raise forms.ValidationError(f"Must be one of: {', '.join(str(item) for item in enum)}.")
-    return value
-
-
-class PluginConfigFormMixin:
-    plugin_groups: list[dict[str, Any]]
-
-    def build_plugin_groups(self, runtime_config: Mapping[str, Any] | None = None) -> None:
-        all_plugins = get_plugins()
-        plugin_configs = discover_plugin_configs()
-        runtime_config = runtime_config or get_config()
-        self.plugin_config_binary_urls = get_plugin_config_binary_urls(runtime_config)
-        grouped_plugins = set().union(*(group[-1] for group in PLUGIN_GROUP_DEFINITIONS))
-        other_plugins = tuple(sorted(set(all_plugins) - grouped_plugins - HIDDEN_PLUGIN_CONFIG_UI_PLUGINS))
-
-        for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
-            if field_name in self.fields:
-                get_choice_field(self, field_name).choices = [
-                    (p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins
-                ]
-
-        if "other_plugins" in self.fields:
-            get_choice_field(self, "other_plugins").choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in other_plugins]
-
-        group_specs = (
-            *PLUGIN_GROUP_DEFINITIONS,
-            ("other_plugins", "Other", "", "", "", other_plugins),
-        )
-        self.plugin_groups = [
-            {
-                "field_name": field_name,
-                "title": title,
-                "note": note,
-                "dom_id": dom_id,
-                "select_all_group": select_all_group,
-                "show_selectors": field_name in self.fields,
-                "plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config),
-            }
-            for field_name, title, note, dom_id, select_all_group, plugin_names in group_specs
-            if any(plugin in all_plugins for plugin in plugin_names)
-        ]
-
-    def _build_plugin_cards(
-        self,
-        field_name: str,
-        plugin_names: Iterable[str],
-        plugin_configs: dict[str, dict[str, Any]],
-        runtime_config: Mapping[str, Any],
-    ) -> list[dict[str, Any]]:
-        if field_name in self.fields:
-            choices = list(get_choice_field(self, field_name).choices)
-            selected_values = set(self.data.getlist(field_name)) if self.is_bound else set(get_choice_field(self, field_name).initial or [])
-        else:
-            all_plugins = get_plugins()
-            choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins]
-            selected_values = set()
-
-        cards = []
-        for index, (plugin_name, label) in enumerate(choices):
-            schema = plugin_configs.get(str(plugin_name), {})
-            properties = schema.get("properties") or {}
-            enabled_config_key = f"{str(plugin_name).upper()}_ENABLED"
-            enabled_prop_schema = properties.get(enabled_config_key)
-            if not isinstance(enabled_prop_schema, dict) or "boolean" not in _schema_types(enabled_prop_schema):
-                enabled_config_key = ""
-            config_fields = [
-                self._build_plugin_config_field(str(plugin_name), str(config_key), prop_schema, runtime_config)
-                for config_key, prop_schema in properties.items()
-                if isinstance(prop_schema, dict)
-            ]
-            cards.append(
-                {
-                    "name": str(plugin_name),
-                    "label": label,
-                    "checked": str(plugin_name) in selected_values,
-                    "checkbox_id": f"id_{field_name}_{index}",
-                    "enabled_config_key": enabled_config_key,
-                    "description": str(schema.get("description") or "").strip(),
-                    "required_plugins": [str(item) for item in schema.get("required_plugins") or []],
-                    "required_binaries_count": len(schema.get("required_binaries") or []),
-                    "config_fields": config_fields,
-                    "config_count": len(config_fields),
-                },
-            )
-        return cards
-
-    def _build_plugin_config_field(
-        self,
-        plugin_name: str,
-        config_key: str,
-        prop_schema: Mapping[str, Any],
-        runtime_config: Mapping[str, Any],
-    ) -> dict[str, Any]:
-        schema_types = _schema_types(prop_schema)
-        enum = prop_schema.get("enum")
-        input_name = _plugin_config_input_name(plugin_name, config_key)
-        current_value = runtime_config.get(config_key, prop_schema.get("default", ""))
-        if self.is_bound and input_name in self.data:
-            try:
-                current_value = _coerce_plugin_config_value(self.data.get(input_name), prop_schema)
-            except (TypeError, ValueError, json.JSONDecodeError, forms.ValidationError):
-                current_value = self.data.get(input_name)
-
-        default_value = prop_schema.get("default", "")
-        fallback_key = prop_schema.get("x-fallback")
-        default_display = f"{{{fallback_key}}}" if fallback_key else default_value
-        is_sensitive = bool(prop_schema.get("x-sensitive"))
-        input_value = "" if is_sensitive else _jsonish(current_value)
-        field_kind = "text"
-        input_type = "text"
-        options = []
-
-        if "boolean" in schema_types:
-            field_kind = "boolean"
-            input_value = "true" if bool(current_value) else "false"
-        elif isinstance(enum, list) and enum:
-            field_kind = "select"
-            options = [
-                {
-                    "value": str(option),
-                    "label": str(option),
-                    "selected": str(option) == str(current_value),
-                }
-                for option in enum
-            ]
-        elif "integer" in schema_types or "number" in schema_types:
-            field_kind = "number"
-            input_type = "number"
-        elif "array" in schema_types or "object" in schema_types:
-            field_kind = "json"
-            input_value = "" if is_sensitive else json.dumps(current_value, indent=2, sort_keys=True, default=str)
-        elif is_sensitive:
-            input_type = "password"
-
-        return {
-            "key": config_key,
-            "input_name": input_name,
-            "kind": field_kind,
-            "input_type": input_type,
-            "value": input_value,
-            "checked": bool(current_value),
-            "options": options,
-            "description": str(prop_schema.get("description") or "").strip(),
-            "default": _jsonish(default_display),
-            "current": "configured" if is_sensitive and current_value else _jsonish(current_value),
-            "current_url": self.plugin_config_binary_urls.get(config_key, "") if str(config_key).endswith("_BINARY") else "",
-            "is_sensitive": is_sensitive,
-            "minimum": prop_schema.get("minimum"),
-            "maximum": prop_schema.get("maximum"),
-            "pattern": prop_schema.get("pattern"),
-            "type_label": " / ".join(schema_types),
-        }
-
-    def clean_plugin_config_overrides(self, effective_config: Mapping[str, Any] | None = None) -> dict[str, Any]:
-        if not self.is_bound:
-            return {}
-
-        effective_config = effective_config or get_config()
-        overrides: dict[str, Any] = {}
-        sources: dict[str, str] = {}
-
-        for plugin_name, schema in discover_plugin_configs().items():
-            for config_key, prop_schema in (schema.get("properties") or {}).items():
-                if not isinstance(prop_schema, dict):
-                    continue
-
-                input_name = _plugin_config_input_name(plugin_name, config_key)
-                if input_name not in self.data:
-                    continue
-
-                raw_value: Any = self.data.get(input_name)
-                if "array" in _schema_types(prop_schema) and isinstance(prop_schema.get("enum"), list):
-                    raw_value = self.data.getlist(input_name)
-
-                if prop_schema.get("x-sensitive") and raw_value == "":
-                    continue
-
-                try:
-                    coerced_value = _coerce_plugin_config_value(raw_value, prop_schema)
-                except (TypeError, ValueError, json.JSONDecodeError) as err:
-                    self.add_error("config", forms.ValidationError(f"{config_key}: {err}"))
-                    continue
-                except forms.ValidationError as err:
-                    self.add_error("config", forms.ValidationError(f"{config_key}: {err.messages[0]}"))
-                    continue
-
-                base_value = effective_config.get(config_key, prop_schema.get("default", ""))
-                if _same_config_value(coerced_value, base_value):
-                    continue
-
-                existing_value = overrides.get(config_key)
-                if config_key in overrides and not _same_config_value(existing_value, coerced_value):
-                    self.add_error(
-                        "config",
-                        forms.ValidationError(
-                            f"{config_key} was set differently under {sources[config_key]} and {plugin_name}. Set it once in Custom config overrides.",
-                        ),
-                    )
-                    continue
-
-                overrides[config_key] = coerced_value
-                sources[config_key] = plugin_name
-
-        return overrides
-
-    def plugin_config_keys(self) -> set[str]:
-        return {
-            str(config_key)
-            for schema in discover_plugin_configs().values()
-            for config_key, prop_schema in (schema.get("properties") or {}).items()
-            if isinstance(prop_schema, dict)
-        }
-
-
-def get_plugin_config_binary_urls(runtime_config: Mapping[str, Any]) -> dict[str, str]:
-    from archivebox.config.views import get_environment_binary_url, get_installed_binary_change_url
-    from archivebox.machine.models import Binary, Machine
-
-    binary_keys = {
-        str(config_key)
-        for schema in discover_plugin_configs().values()
-        for config_key, prop_schema in (schema.get("properties") or {}).items()
-        if isinstance(prop_schema, dict) and str(config_key).endswith("_BINARY")
-    }
-    urls: dict[str, str] = {}
-    machine = Machine.current()
-    for key in binary_keys:
-        value = str(runtime_config.get(key) or "").strip()
-        if not value:
-            continue
-        name = Path(value).name if "/" in value else value
-        binary = Binary.objects.get_valid_binary(value, machine=machine)
-        if binary is None and "/" in value:
-            binary = (
-                Binary.objects.exclude(abspath="")
-                .exclude(abspath__isnull=True)
-                .filter(machine=machine, abspath=value)
-                .order_by("-modified_at")
-                .first()
-            )
-        if binary is None and name != value:
-            binary = Binary.objects.get_valid_binary(name, machine=machine)
-        urls[key] = get_installed_binary_change_url(getattr(binary, "name", name), binary) or get_environment_binary_url(name)
-    return urls
-
 
 class AddLinkForm(PluginConfigFormMixin, forms.Form):
     # Basic fields
@@ -509,6 +38,13 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         widget=forms.Textarea(
             attrs={
                 "data-url-regex": URL_REGEX.pattern,
+                "placeholder": (
+                    "Enter URLs to archive, as one per line, CSV, JSON, or embedded in text "
+                    "(e.g. markdown, HTML, etc.). Examples:\n"
+                    "https://example.com\n"
+                    "https://news.ycombinator.com,https://news.google.com\n"
+                    "[ArchiveBox](https://github.com/ArchiveBox/ArchiveBox)"
+                ),
             },
         ),
         required=True,
@@ -526,7 +62,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         widget=forms.RadioSelect(attrs={"class": "depth-selection"}),
     )
     max_urls = forms.IntegerField(
-        label="Max URLs",
+        label="Max crawl URLs",
         required=False,
         min_value=0,
         initial=0,
@@ -548,6 +84,29 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
+    crawl_timeout = forms.CharField(
+        label="Max crawl time",
+        required=False,
+        initial=0,
+        widget=forms.TextInput(
+            attrs={
+                "pattern": TIMEOUT_INPUT_PATTERN,
+                "title": "Use 0, integer seconds, or a duration like 1.5m or 1hr. Non-zero values must be greater than 10 seconds.",
+                "placeholder": "0, 300, 1.5m, or 1hr",
+            },
+        ),
+    )
+    timeout = forms.CharField(
+        label="Max subtask time",
+        required=False,
+        widget=forms.TextInput(
+            attrs={
+                "pattern": TIMEOUT_INPUT_PATTERN,
+                "title": "Use integer seconds or a duration like 1.5m or 1hr. Non-zero values must be greater than 10 seconds.",
+                "placeholder": "60, 1.5m, or 1hr",
+            },
+        ),
+    )
     snapshot_max_size = forms.CharField(
         label="Max snapshot size",
         required=False,
@@ -558,8 +117,18 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
+    delete_after = forms.CharField(
+        label="Delete after",
+        required=False,
+        initial="0",
+        widget=forms.TextInput(
+            attrs={
+                "placeholder": "0 = keep forever, or e.g. 1d / 6mo",
+            },
+        ),
+    )
     crawl_max_concurrent_snapshots = forms.IntegerField(
-        label="Max concurrent snapshots",
+        label="Max in parallel",
         required=False,
         min_value=1,
         widget=forms.NumberInput(
@@ -641,14 +210,20 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         ),
     )
     persona = forms.ModelChoiceField(
-        label="Persona (authentication profile)",
+        label="Persona (configuration profile)",
         required=False,
         queryset=Persona.objects.none(),
         empty_label=None,
         to_field_name="name",
     )
-    index_only = forms.BooleanField(
-        label="Index only dry run (add crawl but don't archive yet)",
+    permissions = forms.ChoiceField(
+        label="Permissions",
+        choices=PERMISSIONS_CHOICES,
+        initial="public",
+        required=True,
+    )
+    start_paused = forms.BooleanField(
+        label="Start paused",
         initial=False,
         required=False,
     )
@@ -660,21 +235,54 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
     )
 
     def __init__(self, *args, **kwargs):
+        self.request = kwargs.pop("request", None)
+        self.can_override_crawl_config = bool(self.request and is_admin_user(self.request))
         super().__init__(*args, **kwargs)
 
         default_persona = Persona.get_or_create_default()
-        self.fields["persona"].queryset = Persona.objects.order_by("name")
-        self.fields["persona"].initial = default_persona.name
-        self.fields["crawl_max_concurrent_snapshots"].initial = get_config(persona=default_persona).CRAWL_MAX_CONCURRENT_SNAPSHOTS
+        persona_queryset = Persona.objects.order_by("name")
+        if not self.can_override_crawl_config:
+            persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
+        self.fields["persona"].queryset = persona_queryset
+
+        selected_persona = persona_queryset.filter(id=default_persona.id).first() or persona_queryset.first()
+        default_config = get_config(persona=selected_persona) if selected_persona else get_config()
+        if selected_persona:
+            self.fields["persona"].initial = selected_persona.name
+        self.fields["permissions"].initial = default_config.PERMISSIONS
+        self.fields["timeout"].initial = default_config.TIMEOUT
+        self.fields["crawl_max_concurrent_snapshots"].initial = default_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
+        self.fields["delete_after"].initial = default_config.DELETE_AFTER
+        self.fields["url_filters"].initial = {
+            "allowlist": "",
+            "denylist": "",
+            "same_domain_only": False,
+            "subpaths_only": False,
+            "only_new": bool(default_config.ONLY_NEW),
+        }
 
-        selected_persona = default_persona
         if self.is_bound:
-            selected_persona = Persona.objects.filter(name=str(self.data.get(self.add_prefix("persona")) or "")).first() or default_persona
-        self.build_plugin_groups(get_config(persona=selected_persona))
+            selected_persona = (
+                persona_queryset.filter(name=str(self.data.get(self.add_prefix("persona")) or "")).first() or selected_persona
+            )
+        if self.can_override_crawl_config:
+            self.build_plugin_groups(get_config(persona=selected_persona) if selected_persona else get_config())
+        else:
+            all_plugins = get_plugins()
+            for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
+                get_choice_field(self, field_name).choices = [(p, p) for p in all_plugins if p in plugin_names]
+            get_choice_field(self, "other_plugins").choices = [(p, p) for p in all_plugins]
+            self.plugin_groups = []
 
     def clean(self):
         cleaned_data = super().clean() or {}
 
+        if not self.can_override_crawl_config:
+            cleaned_data["plugins"] = []
+            cleaned_data["plugin_config"] = {}
+            cleaned_data["config"] = {}
+            return cleaned_data
+
         # Combine all plugin groups into single list
         all_selected_plugins = []
         for field in [
@@ -719,6 +327,8 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
             "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
             "same_domain_only": bool(value.get("same_domain_only")),
+            "subpaths_only": bool(value.get("subpaths_only")),
+            "only_new": bool(value.get("only_new")),
         }
 
     def clean_max_urls(self):
@@ -737,6 +347,37 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             raise forms.ValidationError("Max crawl size must be 0 or a positive number of bytes.")
         return value
 
+    def clean_crawl_timeout(self):
+        return self._clean_timeout_seconds(self.cleaned_data.get("crawl_timeout"), "Max crawl time", blank_value=0)
+
+    def clean_timeout(self):
+        return self._clean_timeout_seconds(self.cleaned_data.get("timeout"), "Max subtask time", blank_value=None)
+
+    def _clean_timeout_seconds(self, raw_value, field_label: str, *, blank_value):
+        raw_value = str(raw_value or "").strip().lower()
+        if not raw_value:
+            return blank_value
+        if raw_value.isdigit():
+            value = int(raw_value)
+        else:
+            match = re.fullmatch(r"(\d+(?:\.\d+)?)\s*(s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours)", raw_value)
+            if not match:
+                raise forms.ValidationError(f"{field_label} must be seconds or a duration like 1.5m or 1hr.")
+            amount_str, unit = match.groups()
+            try:
+                amount = Decimal(amount_str)
+            except InvalidOperation as err:
+                raise forms.ValidationError(f"{field_label} must be seconds or a duration like 1.5m or 1hr.") from err
+            multiplier = 1
+            if unit in {"m", "min", "mins", "minute", "minutes"}:
+                multiplier = 60
+            elif unit in {"h", "hr", "hrs", "hour", "hours"}:
+                multiplier = 60 * 60
+            value = int((amount * multiplier).to_integral_value(rounding=ROUND_CEILING))
+        if 0 < value <= 10:
+            raise forms.ValidationError(f"{field_label} must be 0 or greater than 10 seconds.")
+        return value
+
     def clean_snapshot_max_size(self):
         raw_value = str(self.cleaned_data.get("snapshot_max_size") or "").strip()
         if not raw_value:
@@ -749,6 +390,14 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             raise forms.ValidationError("Max snapshot size must be 0 or a positive number of bytes.")
         return value
 
+    def clean_delete_after(self):
+        raw_value = str(self.cleaned_data.get("delete_after") or "0").strip() or "0"
+        try:
+            parse_delete_after(raw_value)
+        except ValueError as err:
+            raise forms.ValidationError(str(err))
+        return raw_value
+
     def clean_crawl_max_concurrent_snapshots(self):
         value = self.cleaned_data.get("crawl_max_concurrent_snapshots")
         if value in (None, ""):

archivebox/core/host_utils.py

@@ -1,259 +0,0 @@
-from __future__ import annotations
-
-import re
-from typing import Any
-from urllib.parse import urlparse
-
-from archivebox.config.common import get_config
-
-
-_SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
-_SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P<suffix>[0-9a-fA-F]{12})$")
-
-
-def split_host_port(host: str) -> tuple[str, str | None]:
-    parsed = urlparse(f"//{host}")
-    hostname = (parsed.hostname or host or "").lower()
-    port = str(parsed.port) if parsed.port else None
-    return hostname, port
-
-
-def _normalize_base_url(value: str | None) -> str:
-    if not value:
-        return ""
-    base = value.strip()
-    if not base:
-        return ""
-    if "://" not in base:
-        base = f"http://{base}"
-    parsed = urlparse(base)
-    if not parsed.netloc:
-        return ""
-    return f"{parsed.scheme}://{parsed.netloc}"
-
-
-def normalize_base_url(value: str | None) -> str:
-    return _normalize_base_url(value)
-
-
-def get_listen_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    return (config.LISTEN_HOST or "").strip()
-
-
-def get_listen_parts(config: dict[str, Any] | None = None, **config_kwargs: Any) -> tuple[str, str | None]:
-    config = config or get_config(**config_kwargs)
-    return split_host_port(get_listen_host(config=config))
-
-
-def _build_listen_host(subdomain: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    host, port = get_listen_parts(config=config)
-    if not host:
-        return ""
-    full_host = f"{subdomain}.{host}" if subdomain else host
-    if port:
-        return f"{full_host}:{port}"
-    return full_host
-
-
-def get_admin_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    override = _normalize_base_url(config.ADMIN_BASE_URL)
-    if override:
-        return urlparse(override).netloc.lower()
-    return _build_listen_host("admin", config=config)
-
-
-def get_web_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    override = _normalize_base_url(config.ARCHIVE_BASE_URL)
-    if override:
-        return urlparse(override).netloc.lower()
-    return _build_listen_host("web", config=config)
-
-
-def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    return _build_listen_host("api", config=config)
-
-
-def get_public_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    return _build_listen_host("public", config=config)
-
-
-def get_snapshot_subdomain(snapshot_id: str) -> str:
-    normalized = re.sub(r"[^0-9a-fA-F]", "", snapshot_id or "")
-    suffix = (normalized[-12:] if len(normalized) >= 12 else normalized).lower()
-    return f"snap-{suffix}"
-
-
-def get_snapshot_host(snapshot_id: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    return _build_listen_host(get_snapshot_subdomain(snapshot_id), config=config)
-
-
-def get_original_host(domain: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    return _build_listen_host(domain, config=config)
-
-
-def is_snapshot_subdomain(subdomain: str) -> bool:
-    value = (subdomain or "").strip()
-    return bool(_SNAPSHOT_SUBDOMAIN_RE.match(value) or _SNAPSHOT_ID_RE.match(value))
-
-
-def get_snapshot_lookup_key(snapshot_ref: str) -> str:
-    value = (snapshot_ref or "").strip().lower()
-    match = _SNAPSHOT_SUBDOMAIN_RE.match(value)
-    if match:
-        return match.group("suffix")
-    return value
-
-
-def get_listen_subdomain(request_host: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return ""
-    req_host, req_port = split_host_port(request_host)
-    listen_host, listen_port = get_listen_parts(config=config)
-    if not listen_host:
-        return ""
-    if listen_port and req_port and listen_port != req_port:
-        return ""
-    if req_host == listen_host:
-        return ""
-    suffix = f".{listen_host}"
-    if req_host.endswith(suffix):
-        return req_host[: -len(suffix)]
-    return ""
-
-
-def host_matches(request_host: str, target_host: str) -> bool:
-    if not request_host or not target_host:
-        return False
-    req_host, req_port = split_host_port(request_host)
-    target_host_only, target_port = split_host_port(target_host)
-    if req_host != target_host_only:
-        return False
-    if target_port and req_port and target_port != req_port:
-        return False
-    return True
-
-
-def _scheme_from_request(request=None, config: dict[str, Any] | None = None) -> str:
-    if request and request.scheme != "http":
-        return request.scheme
-    config = config or get_config()
-    for base_url in (config.ARCHIVE_BASE_URL, config.ADMIN_BASE_URL):
-        override = _normalize_base_url(base_url)
-        if override:
-            return urlparse(override).scheme
-    if request:
-        return request.scheme
-    return "http"
-
-
-def _build_base_url_for_host(host: str, request=None, config: dict[str, Any] | None = None) -> str:
-    if not host:
-        return ""
-    scheme = _scheme_from_request(request, config=config)
-    return f"{scheme}://{host}"
-
-
-def get_admin_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    override = _normalize_base_url(config.ADMIN_BASE_URL)
-    if override:
-        return override
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_base_url_for_host(get_listen_host(config=config), request=request, config=config)
-    return _build_base_url_for_host(get_admin_host(config=config), request=request, config=config)
-
-
-def get_web_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    override = _normalize_base_url(config.ARCHIVE_BASE_URL)
-    if override:
-        return override
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_base_url_for_host(get_listen_host(config=config), request=request, config=config)
-    return _build_base_url_for_host(get_web_host(config=config), request=request, config=config)
-
-
-def get_api_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_base_url_for_host(get_listen_host(config=config), request=request, config=config)
-    return _build_base_url_for_host(get_api_host(config=config), request=request, config=config)
-
-
-def get_public_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    return _build_base_url_for_host(get_public_host(config=config), request=request, config=config)
-
-
-# Backwards-compat aliases (archive == web)
-def get_archive_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return get_web_base_url(request=request, config=config, **config_kwargs)
-
-
-def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_url(get_web_base_url(request=request, config=config), f"/snapshot/{snapshot_id}")
-    return _build_base_url_for_host(get_snapshot_host(snapshot_id, config=config), request=request, config=config)
-
-
-def get_original_base_url(domain: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_url(get_web_base_url(request=request, config=config), f"/original/{domain}")
-    return _build_base_url_for_host(get_original_host(domain, config=config), request=request, config=config)
-
-
-def build_admin_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_admin_base_url(request, config=config, **config_kwargs), path)
-
-
-def build_web_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_web_base_url(request, config=config, **config_kwargs), path)
-
-
-def build_api_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_api_base_url(request, config=config, **config_kwargs), path)
-
-
-def build_archive_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_archive_base_url(request, config=config, **config_kwargs), path)
-
-
-def build_snapshot_url(snapshot_id: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_snapshot_base_url(snapshot_id, request=request, config=config, **config_kwargs), path)
-
-
-def build_original_url(domain: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_original_base_url(domain, request=request, config=config, **config_kwargs), path)
-
-
-def _build_url(base_url: str, path: str) -> str:
-    if not base_url:
-        if not path:
-            return ""
-        return path if path.startswith("/") else f"/{path}"
-    if not path:
-        return base_url
-    return f"{base_url}{path if path.startswith('/') else f'/{path}'}"

archivebox/core/middleware.py

@@ -15,12 +15,13 @@ from django.http import HttpResponseForbidden, HttpResponseNotModified
 from archivebox.config.common import get_config
 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH
-from archivebox.core.host_utils import (
+from archivebox.core.routes_util import (
     build_snapshot_url,
     build_admin_url,
     build_web_url,
     get_api_host,
     get_admin_host,
+    get_base_host,
     get_listen_host,
     get_listen_subdomain,
     get_public_host,
@@ -36,14 +37,32 @@ ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in"
 
 
 def _admin_login_hint_cookie_domain(config) -> str | None:
+    """Resolve the parent domain to scope the cross-subdomain login hint.
+
+    NOTE: this cookie carries only the single bit "user is logged in on
+    admin somewhere"; it MUST NOT be confused with the session cookie,
+    which stays admin-host-scoped (see core/settings.py
+    SESSION_COOKIE_DOMAIN comment — admin/public is a security boundary).
+
+    Returns the hostname portion of ``get_base_host`` (which respects
+    ``BASE_URL`` and falls back to the local-bind mapping). Strips the
+    port — cookie ``Domain=`` attributes don't include ports. Returns
+    ``None`` when subdomain routing is off, the base host is empty, or
+    the base host is an IP / bare ``localhost`` (browsers reject
+    cross-host cookies for those).
+    """
     if not config.USES_SUBDOMAIN_ROUTING:
         return None
-    listen_host, _listen_port = split_host_port(get_listen_host(config=config))
+    base_host = get_base_host(config=config)
+    if not base_host:
+        return None
+    hostname, _port = split_host_port(base_host)
+    if not hostname or hostname == "localhost":
+        return None
     try:
-        ipaddress.ip_address(listen_host)
+        ipaddress.ip_address(hostname)
     except ValueError:
-        if listen_host and listen_host != "localhost":
-            return listen_host
+        return hostname
     return None
 
 
@@ -104,7 +123,7 @@ def CacheControlMiddleware(get_response):
                 if config is None:
                     config = get_config(resolve_plugins=False)
                     request.archivebox_config = config
-                policy = "public" if config.PUBLIC_SNAPSHOTS else "private"
+                policy = "private" if config.PERMISSIONS == "private" else "public"
                 response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
                 # print('Set Cache-Control header to', response['Cache-Control'])
         return response
@@ -146,6 +165,9 @@ def HostRoutingMiddleware(get_response):
     )
 
     def middleware(request):
+        if request.path in {"/health", "/health/"}:
+            return get_response(request)
+
         request_host = (request.get_host() or "").lower()
         config = getattr(request, "archivebox_config", None)
         if config is None:
@@ -163,12 +185,32 @@ def HostRoutingMiddleware(get_response):
         if request.path.startswith("/static/") or request.path in {"/favicon.ico", "/robots.txt"}:
             return get_response(request)
 
+        # In subdomain mode with no explicit BASE_URL we can't safely emit
+        # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
+        # request's own Host (via the request-host fallback in get_base_url),
+        # so prepending ``admin.`` to whatever the client sent produces a
+        # redirect chain of ``admin.admin.admin.<host>``. Pass the request
+        # through; the misconfig banner on the rendered page tells the user
+        # to pin BASE_URL so the redirects can resume.
+        if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
+            return get_response(request)
+
         if config.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host):
+            # ``/add`` is admin-only unless ``PUBLIC_ADD_VIEW`` is on. Without
+            # this redirect, hitting it on public.* falls into AddView's
+            # auth check, bounces through ``/accounts/login/?next=/add/`` →
+            # ``/admin/login/?next=/add/``, and Django admin's LoginView
+            # silently drops ``next`` when the user already has an admin
+            # session — dumping the user on the admin homepage instead of
+            # the add form. Routing the request to admin.* directly lets
+            # AddView run on the host where the session lives.
+            add_should_redirect = not config.PUBLIC_ADD_VIEW and (request.path == "/add" or request.path.startswith("/add/"))
             if (
                 request.path == "/admin"
                 or request.path.startswith("/admin/")
                 or request.path == "/accounts"
                 or request.path.startswith("/accounts/")
+                or add_should_redirect
             ):
                 target = build_admin_url(request.path, request=request)
                 if request.META.get("QUERY_STRING"):
@@ -259,7 +301,13 @@ def HostRoutingMiddleware(get_response):
                 target = f"{target}?{request.META['QUERY_STRING']}"
             return redirect(target)
 
-        if admin_host or web_host:
+        if (admin_host or web_host) and config.BASE_URL:
+            # Only force a canonical-host redirect when BASE_URL was set
+            # explicitly. If BASE_URL is empty (e.g. 0.7.3 → 0.9.0 upgrade
+            # where the user has CSRF_TRUSTED_ORIGINS but never set BASE_URL),
+            # the subdomain we'd redirect to may not actually resolve in the
+            # user's reverse proxy — serve the request as-is instead and let
+            # the misconfig banner surface the problem in the page.
             target = build_web_url(request.path, request=request)
             if target:
                 if request.META.get("QUERY_STRING"):

archivebox/core/migrations/0023_upgrade_to_0_9_0.py

@@ -9,6 +9,9 @@ from datetime import datetime
 import json
 
 
+PROGRESS_EVERY = 10000
+
+
 def get_table_columns(table_name):
     """Get list of column names for a table."""
     cursor = connection.cursor()
@@ -64,7 +67,9 @@ def upgrade_core_tables(apps, schema_editor):
     has_uuid = "uuid" in archiveresult_cols
     has_abid = "abid" in archiveresult_cols
 
-    print(f"DEBUG: ArchiveResult row_count={row_count}, has_data={has_data}, has_uuid={has_uuid}, has_abid={has_abid}")
+    if has_data:
+        source_schema = "0.8.x abid" if has_abid and not has_uuid else "0.8.x uuid" if has_uuid else "0.7.x"
+        print(f"    - Rebuilding core tables from {source_schema} schema ({row_count} ArchiveResults)...")
 
     # ============================================================================
     # PART 1: Upgrade core_archiveresult table
@@ -113,7 +118,7 @@ def upgrade_core_tables(apps, schema_editor):
 
         if has_uuid and not has_abid:
             # Migrating from v0.7.2+ (has uuid column)
-            print("Migrating ArchiveResult from v0.7.2+ schema (with uuid)...")
+            print(f"      copying {row_count} ArchiveResults...")
             select_cols = ["id", "uuid", "snapshot_id", "cmd", "pwd", "cmd_version", "start_ts", "end_ts", "status", "extractor", "output"]
             if has_archiveresult_created_at:
                 select_cols.append("created_at")
@@ -121,7 +126,7 @@ def upgrade_core_tables(apps, schema_editor):
                 select_cols.append("modified_at")
             cursor.execute(f"SELECT {', '.join(select_cols)} FROM core_archiveresult")
             old_records = cursor.fetchall()
-            for record in old_records:
+            for i, record in enumerate(old_records, start=1):
                 values = dict(zip(select_cols, record))
                 try:
                     new_uuid = UUID(str(values["uuid"])).hex
@@ -153,12 +158,14 @@ def upgrade_core_tables(apps, schema_editor):
                         values.get("modified_at") or end_ts,
                     ),
                 )
+                if i % PROGRESS_EVERY == 0:
+                    print(f"        copied {i}/{len(old_records)} ArchiveResults...")
         elif has_abid and not has_uuid:
             # Migrating from v0.8.6rc0 (has abid instead of uuid)
-            print("Migrating ArchiveResult from v0.8.6rc0 schema...")
+            print(f"      copying {row_count} ArchiveResults...")
             cursor.execute(f"SELECT {', '.join(archiveresult_select_cols)} FROM core_archiveresult")
             old_records = cursor.fetchall()
-            for record in old_records:
+            for i, record in enumerate(old_records, start=1):
                 values = dict(zip(archiveresult_select_cols, record))
                 try:
                     new_uuid = UUID(str(values["id"])).hex
@@ -189,12 +196,14 @@ def upgrade_core_tables(apps, schema_editor):
                         values.get("modified_at") or end_ts,
                     ),
                 )
+                if i % PROGRESS_EVERY == 0:
+                    print(f"        copied {i}/{len(old_records)} ArchiveResults...")
         else:
             # Migrating from v0.7.2 (no uuid or abid column - generate fresh UUIDs)
-            print("Migrating ArchiveResult from v0.7.2 schema (no uuid - generating UUIDs)...")
+            print(f"      copying {row_count} ArchiveResults...")
             cursor.execute(f"SELECT {', '.join(archiveresult_select_cols)} FROM core_archiveresult")
             old_records = cursor.fetchall()
-            for record in old_records:
+            for i, record in enumerate(old_records, start=1):
                 values = dict(zip(archiveresult_select_cols, record))
                 new_uuid = uuid7().hex
                 start_ts = values["start_ts"] or datetime.now().isoformat()
@@ -223,6 +232,9 @@ def upgrade_core_tables(apps, schema_editor):
                         values.get("modified_at") or end_ts,
                     ),
                 )
+                if i % PROGRESS_EVERY == 0:
+                    print(f"        copied {i}/{len(old_records)} ArchiveResults...")
+        print(f"      copied {len(old_records)} ArchiveResults")
 
     cursor.execute("DROP TABLE IF EXISTS core_archiveresult;")
     cursor.execute("ALTER TABLE core_archiveresult_new RENAME TO core_archiveresult;")
@@ -279,7 +291,7 @@ def upgrade_core_tables(apps, schema_editor):
 
             if has_added and not has_bookmarked_at:
                 # Migrating from v0.7.2 (has added/updated fields)
-                print("Migrating Snapshot from v0.7.2 schema...")
+                print("      copying Snapshots from 0.7.x schema...")
                 # timestamp is the legacy bookmark/import timestamp and archive/{timestamp} identity.
                 # added is the DB row creation/import time, and updated was renamed to downloaded_at in 0.8.x.
                 cursor.execute("""
@@ -301,12 +313,26 @@ def upgrade_core_tables(apps, schema_editor):
                         COALESCE(added, CURRENT_TIMESTAMP) as created_at,
                         COALESCE(updated, added, CURRENT_TIMESTAMP) as modified_at,
                         updated as downloaded_at,
-                        'queued' as status
+                        CASE
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                                AND core_archiveresult.status IN ('queued', 'started', 'backoff')
+                            )
+                            THEN 'queued'
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                            )
+                            THEN 'sealed'
+                            ELSE 'queued'
+                        END as status
                     FROM core_snapshot;
                 """)
+                print(f"      copied {cursor.rowcount} Snapshots")
             elif has_bookmarked_at and not has_added:
                 # Migrating from v0.8.6rc0 (already has bookmarked_at/created_at/modified_at)
-                print("Migrating Snapshot from v0.8.6rc0 schema...")
+                print("      copying Snapshots from 0.8.x schema...")
                 # Check what fields exist
                 has_status = "status" in snapshot_cols
                 has_retry_at = "retry_at" in snapshot_cols
@@ -320,7 +346,25 @@ def upgrade_core_tables(apps, schema_editor):
                     select_cols.append("REPLACE(crawl_id, '-', '')")
                 if has_status:
                     insert_cols.append("status")
-                    select_cols.append("status")
+                    select_cols.append(
+                        """
+                        CASE
+                            WHEN status IN ('sealed', 'started', 'paused') THEN status
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                                AND core_archiveresult.status IN ('queued', 'started', 'backoff')
+                            )
+                            THEN status
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                            )
+                            THEN 'sealed'
+                            ELSE status
+                        END
+                        """,
+                    )
                 if has_retry_at:
                     insert_cols.append("retry_at")
                     select_cols.append("retry_at")
@@ -330,6 +374,7 @@ def upgrade_core_tables(apps, schema_editor):
                     SELECT {", ".join(select_cols)}
                     FROM core_snapshot;
                 """)
+                print(f"      copied {cursor.rowcount} Snapshots")
             else:
                 print(f"Warning: Unexpected Snapshot schema - has_added={has_added}, has_bookmarked_at={has_bookmarked_at}")
 
@@ -380,7 +425,7 @@ def upgrade_core_tables(apps, schema_editor):
 
             if tag_id_type and "char" in tag_id_type.lower():
                 # v0.8.6rc0: Tag IDs are UUIDs, need to convert to INTEGER
-                print("Converting Tag IDs from UUID to INTEGER...")
+                print("      converting Tag IDs from UUID to integers...")
 
                 # Get all tags with their UUIDs
                 cursor.execute("SELECT id, name, slug, created_at, modified_at, created_by_id FROM core_tag ORDER BY name")
@@ -399,6 +444,8 @@ def upgrade_core_tables(apps, schema_editor):
                     """,
                         (i, name, slug, created_at, modified_at, created_by_id),
                     )
+                    if i % PROGRESS_EVERY == 0:
+                        print(f"        copied {i}/{len(tags)} Tags...")
 
                 # Update snapshot_tags to use new INTEGER IDs
                 cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_snapshot_tags'")
@@ -410,7 +457,7 @@ def upgrade_core_tables(apps, schema_editor):
                     cursor.execute("DELETE FROM core_snapshot_tags")
 
                     # Re-insert with new integer tag IDs
-                    for st_id, snapshot_id, old_tag_id in snapshot_tags:
+                    for i, (st_id, snapshot_id, old_tag_id) in enumerate(snapshot_tags, start=1):
                         new_tag_id = uuid_to_int_map.get(old_tag_id)
                         if new_tag_id:
                             cursor.execute(
@@ -420,14 +467,18 @@ def upgrade_core_tables(apps, schema_editor):
                             """,
                                 (st_id, snapshot_id, new_tag_id),
                             )
+                        if i % PROGRESS_EVERY == 0:
+                            print(f"        copied {i}/{len(snapshot_tags)} SnapshotTag rows...")
+                print(f"      copied {len(tags)} Tags")
             else:
                 # v0.7.2: Tag IDs are already INTEGER
-                print("Migrating Tag from v0.7.2 schema...")
+                print("      copying Tags from 0.7.x schema...")
                 cursor.execute("""
                     INSERT OR IGNORE INTO core_tag_new (id, name, slug)
                     SELECT id, name, slug
                     FROM core_tag;
                 """)
+                print(f"      copied {cursor.rowcount} Tags")
 
     cursor.execute("DROP TABLE IF EXISTS core_tag;")
     cursor.execute("ALTER TABLE core_tag_new RENAME TO core_tag;")
@@ -437,7 +488,7 @@ def upgrade_core_tables(apps, schema_editor):
     cursor.execute("CREATE INDEX IF NOT EXISTS core_tag_created_by_id_idx ON core_tag(created_by_id);")
 
     if has_data:
-        print("✓ Core tables upgraded to v0.9.0")
+        print("    ✓ Core table rebuild complete")
 
 
 class Migration(migrations.Migration):

archivebox/core/migrations/0025_alter_archiveresult_options_alter_snapshot_options_and_more.py

@@ -40,12 +40,7 @@ def copy_old_fields_to_new(apps, schema_editor):
 
     # NOTE: Snapshot timestamps (added→bookmarked_at, updated→modified_at) were already
     # transformed by migration 0023, so we don't need to copy them here.
-    # NOTE: UUIDs are already populated by migration 0023 for all migration paths
-
-    # Debug: Check Snapshot timestamps at end of RunPython
-    cursor.execute("SELECT id, bookmarked_at, modified_at FROM core_snapshot LIMIT 2")
-    snap_after = cursor.fetchall()
-    print(f"DEBUG 0025: Snapshot timestamps at END of RunPython: {snap_after}")
+    # NOTE: UUIDs are already populated by migration 0023 for all migration paths.
 
 
 class Migration(migrations.Migration):
@@ -151,70 +146,77 @@ class Migration(migrations.Migration):
             field=models.DateTimeField(blank=True, db_index=True, default=django.utils.timezone.now, null=True),
         ),
         # NOTE: bookmarked_at, created_at, and downloaded_at already added by migration 0023
-        migrations.AddField(
-            model_name="snapshot",
-            name="config",
-            field=models.JSONField(default=dict),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="current_step",
-            field=models.PositiveSmallIntegerField(
-                db_index=True,
-                default=0,
-                help_text="Current hook step being executed (0-9). Used for sequential hook execution.",
-            ),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="depth",
-            field=models.PositiveSmallIntegerField(db_index=True, default=0),
-        ),
-        # NOTE: downloaded_at already added by migration 0023 so it can preserve
-        # v0.7.x updated / v0.8.x downloaded_at values without a duplicate table rebuild.
-        # NOTE: fs_version already added by migration 0023 with default='0.8.0'
-        # NOTE: modified_at already added by migration 0023
-        migrations.AddField(
-            model_name="snapshot",
-            name="notes",
-            field=models.TextField(blank=True, default=""),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="num_uses_failed",
-            field=models.PositiveIntegerField(default=0),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="num_uses_succeeded",
-            field=models.PositiveIntegerField(default=0),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="parent_snapshot",
-            field=models.ForeignKey(
-                blank=True,
-                help_text="Parent snapshot that discovered this URL (for recursive crawling)",
-                null=True,
-                on_delete=django.db.models.deletion.SET_NULL,
-                related_name="child_snapshots",
-                to="core.snapshot",
-            ),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="retry_at",
-            field=models.DateTimeField(blank=True, db_index=True, default=django.utils.timezone.now, null=True),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="status",
-            field=models.CharField(
-                choices=[("queued", "Queued"), ("started", "Started"), ("sealed", "Sealed")],
-                db_index=True,
-                default="queued",
-                max_length=15,
-            ),
+        # 0023 rebuilds core_snapshot with these columns already present so it
+        # can preserve legacy status/retry_at/config data in one tight SQLite
+        # table copy. Keep this migration state-only: running normal AddField
+        # operations here causes SQLite to rebuild the table from the pre-0025
+        # state and repopulate existing columns with defaults, reopening sealed
+        # migrated rows as queued.
+        migrations.SeparateDatabaseAndState(
+            database_operations=[],
+            state_operations=[
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="config",
+                    field=models.JSONField(default=dict),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="current_step",
+                    field=models.PositiveSmallIntegerField(
+                        db_index=True,
+                        default=0,
+                        help_text="Current hook step being executed (0-9). Used for sequential hook execution.",
+                    ),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="depth",
+                    field=models.PositiveSmallIntegerField(db_index=True, default=0),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="notes",
+                    field=models.TextField(blank=True, default=""),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="num_uses_failed",
+                    field=models.PositiveIntegerField(default=0),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="num_uses_succeeded",
+                    field=models.PositiveIntegerField(default=0),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="parent_snapshot",
+                    field=models.ForeignKey(
+                        blank=True,
+                        help_text="Parent snapshot that discovered this URL (for recursive crawling)",
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="child_snapshots",
+                        to="core.snapshot",
+                    ),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="retry_at",
+                    field=models.DateTimeField(blank=True, db_index=True, default=django.utils.timezone.now, null=True),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="status",
+                    field=models.CharField(
+                        choices=[("queued", "Queued"), ("started", "Started"), ("sealed", "Sealed")],
+                        db_index=True,
+                        default="queued",
+                        max_length=15,
+                    ),
+                ),
+            ],
         ),
         migrations.AddField(
             model_name="tag",

archivebox/core/migrations/0027_copy_archiveresult_to_process.py

@@ -7,6 +7,9 @@ from pathlib import Path
 from archivebox.uuid_compat import uuid7
 
 
+PROGRESS_EVERY = 10000
+
+
 def parse_cmd_field(cmd_raw):
     """
     Parse cmd field which could be:
@@ -140,35 +143,33 @@ def get_or_create_binary(cursor, machine_id, name, abspath, version):
     cursor.execute("PRAGMA table_info(machine_binary)")
     binary_cols = {row[1] for row in cursor.fetchall()}
 
-    # Use only columns that exist in current schema
-    # 0.8.x schema: id, created_at, modified_at, machine_id, name, binprovider, abspath, version, sha256, num_uses_failed, num_uses_succeeded
-    # 0.9.x schema adds: binproviders, overrides, status, retry_at, output_dir
-    if "binproviders" in binary_cols:
-        # 0.9.x schema
-        cursor.execute(
-            """
-            INSERT INTO machine_binary (
-                id, created_at, modified_at, machine_id,
-                name, binproviders, overrides, binprovider, abspath, version, sha256,
-                status, retry_at, output_dir,
-                num_uses_failed, num_uses_succeeded
-            ) VALUES (?, ?, ?, ?, ?, 'env', '{}', 'env', ?, ?, '',
-                      'succeeded', NULL, '', 0, 0)
+    values_by_col = {
+        "id": binary_id,
+        "created_at": now,
+        "modified_at": now,
+        "machine_id": machine_id,
+        "name": name,
+        "binproviders": "env",
+        "overrides": "{}",
+        "binprovider": "env",
+        "abspath": abspath,
+        "version": version,
+        "sha256": "",
+        "status": "installed",
+        "retry_at": None,
+        "output_dir": "",
+        "num_uses_failed": 0,
+        "num_uses_succeeded": 0,
+    }
+    insert_cols = [col for col in values_by_col if col in binary_cols]
+    placeholders = ", ".join(["?"] * len(insert_cols))
+    cursor.execute(
+        f"""
+        INSERT INTO machine_binary ({", ".join(insert_cols)})
+        VALUES ({placeholders})
         """,
-            [binary_id, now, now, machine_id, name, abspath, version],
-        )
-    else:
-        # 0.8.x schema (simpler)
-        cursor.execute(
-            """
-            INSERT INTO machine_binary (
-                id, created_at, modified_at, machine_id,
-                name, binprovider, abspath, version, sha256,
-                num_uses_failed, num_uses_succeeded
-            ) VALUES (?, ?, ?, ?, ?, 'env', ?, ?, '', 0, 0)
-        """,
-            [binary_id, now, now, machine_id, name, abspath, version],
-        )
+        [values_by_col[col] for col in insert_cols],
+    )
 
     return binary_id
 
@@ -214,36 +215,42 @@ def create_process(cursor, machine_id, pwd, cmd, status, exit_code, started_at,
     # Set retry_at to now for queued processes, NULL otherwise
     retry_at = now if status == "queued" else None
 
+    cursor.execute("PRAGMA table_info(machine_process)")
+    process_cols = {row[1] for row in cursor.fetchall()}
+    values_by_col = {
+        "id": process_id,
+        "created_at": now,
+        "modified_at": now,
+        "machine_id": machine_id,
+        "parent_id": None,
+        "process_type": "hook",
+        "worker_type": "",
+        "pwd": pwd,
+        "cmd": cmd_json,
+        "env": "{}",
+        "timeout": 120,
+        "pid": None,
+        "exit_code": exit_code,
+        "stdout": "",
+        "stderr": "",
+        "started_at": started_at,
+        "ended_at": ended_at,
+        "binary_id": binary_id,
+        "iface_id": None,
+        "url": None,
+        "status": status,
+        "retry_at": retry_at,
+        "num_uses_failed": 0,
+        "num_uses_succeeded": 0,
+    }
+    insert_cols = [col for col in values_by_col if col in process_cols]
+    placeholders = ", ".join(["?"] * len(insert_cols))
     cursor.execute(
-        """
-        INSERT INTO machine_process (
-            id, created_at, modified_at, machine_id, parent_id, process_type,
-            pwd, cmd, env, timeout,
-            pid, exit_code, stdout, stderr,
-            started_at, ended_at,
-            binary_id, iface_id, url,
-            status, retry_at
-        ) VALUES (?, ?, ?, ?, NULL, 'cli',
-                  ?, ?, '{}', 120,
-                  NULL, ?, '', '',
-                  ?, ?,
-                  ?, NULL, NULL,
-                  ?, ?)
-    """,
-        [
-            process_id,
-            now,
-            now,
-            machine_id,
-            pwd,
-            cmd_json,
-            exit_code,
-            started_at,
-            ended_at,
-            binary_id,
-            status,
-            retry_at,
-        ],
+        f"""
+        INSERT INTO machine_process ({", ".join(insert_cols)})
+        VALUES ({placeholders})
+        """,
+        [values_by_col[col] for col in insert_cols],
     )
 
     return process_id
@@ -274,13 +281,8 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
     cursor.execute("PRAGMA table_info(core_archiveresult)")
     cols = {row[1] for row in cursor.fetchall()}
 
-    print(f"DEBUG 0027: Columns found: {sorted(cols)}")
-    print(
-        f"DEBUG 0027: Has cmd={('cmd' in cols)}, pwd={('pwd' in cols)}, cmd_version={('cmd_version' in cols)}, process_id={('process_id' in cols)}",
-    )
-
     if "cmd" not in cols or "pwd" not in cols or "cmd_version" not in cols:
-        print("✓ Fresh install or fields already removed - skipping data copy")
+        print("    ✓ ArchiveResult process metadata already migrated")
         return
 
     # Check if process_id field exists (should exist from 0026)
@@ -304,10 +306,10 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
     results = cursor.fetchall()
 
     if not results:
-        print("✓ No ArchiveResults need Process migration")
+        print("    ✓ No ArchiveResults need Process migration")
         return
 
-    print(f"Migrating {len(results)} ArchiveResults to Process records...")
+    print(f"    - Migrating {len(results)} ArchiveResults to Process rows...")
 
     migrated_count = 0
     skipped_count = 0
@@ -316,16 +318,10 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
     for i, row in enumerate(results):
         ar_id, snapshot_id, plugin, cmd_raw, pwd, cmd_version, status, start_ts, end_ts, created_at = row
 
-        if i == 0:
-            print(f"DEBUG 0027: First row: ar_id={ar_id}, plugin={plugin}, cmd={cmd_raw[:50] if cmd_raw else None}, status={status}")
-
         try:
             # Parse cmd field
             cmd_array = parse_cmd_field(cmd_raw)
 
-            if i == 0:
-                print(f"DEBUG 0027: Parsed cmd: {cmd_array}")
-
             # Extract binary info from cmd[0] if available
             binary_id = None
             if cmd_array and cmd_array[0]:
@@ -342,9 +338,6 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
                     binary_version,
                 )
 
-                if i == 0:
-                    print(f"DEBUG 0027: Created Binary: id={binary_id}, name={binary_name}")
-
             # Map status
             process_status, exit_code = map_status(status)
 
@@ -365,9 +358,6 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
                 binary_id=binary_id,
             )
 
-            if i == 0:
-                print(f"DEBUG 0027: Created Process: id={process_id}")
-
             # Link ArchiveResult to Process
             cursor.execute(
                 "UPDATE core_archiveresult SET process_id = ? WHERE id = ?",
@@ -375,9 +365,8 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
             )
 
             migrated_count += 1
-
-            if i == 0:
-                print("DEBUG 0027: Linked ArchiveResult to Process")
+            if migrated_count % PROGRESS_EVERY == 0:
+                print(f"      migrated {migrated_count}/{len(results)} ArchiveResults...")
 
         except Exception as e:
             print(f"✗ Error migrating ArchiveResult {ar_id}: {e}")
@@ -387,7 +376,7 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
             error_count += 1
             continue
 
-    print(f"✓ Migration complete: {migrated_count} migrated, {skipped_count} skipped, {error_count} errors")
+    print(f"    ✓ Process migration complete: {migrated_count} migrated, {skipped_count} skipped, {error_count} errors")
 
 
 class Migration(migrations.Migration):

archivebox/core/migrations/0029_migrate_archiveresult_to_uuid_pk.py

@@ -6,6 +6,9 @@ from uuid import UUID
 from archivebox.uuid_compat import uuid7
 
 
+PROGRESS_EVERY = 10000
+
+
 def migrate_archiveresult_id_to_uuid(apps, schema_editor):
     """
     Migrate ArchiveResult from integer PK to UUID PK (clean one-step migration).
@@ -28,7 +31,7 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
     # Check if table exists and has data
     cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_archiveresult'")
     if not cursor.fetchone():
-        print("ArchiveResult table does not exist, skipping migration")
+        print("    ✓ ArchiveResult table does not exist, skipping UUID PK migration")
         return
 
     cursor.execute("SELECT COUNT(*) FROM core_archiveresult")
@@ -38,16 +41,16 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
     # (fresh installs create table with uuid from 0025, but model expects no uuid after 0029)
 
     if row_count == 0:
-        print("[0029] Recreating ArchiveResult table schema (integer→UUID PK, removing uuid column)...")
+        print("    - Rebuilding empty ArchiveResult table with UUID primary keys...")
     else:
-        print(f"[0029] Migrating {row_count} ArchiveResult records from integer PK to UUID PK...")
+        print(f"    - Migrating {row_count} ArchiveResults from integer IDs to UUID primary keys...")
 
     # Step 0: Check if machine_process table exists, if not NULL out process_id values
     cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='machine_process'")
     machine_process_exists = cursor.fetchone() is not None
 
     if not machine_process_exists:
-        print("machine_process table does not exist yet, setting process_id to NULL")
+        print("      machine_process is unavailable; clearing process_id references...")
         cursor.execute("UPDATE core_archiveresult SET process_id = NULL WHERE process_id IS NOT NULL")
 
     # Step 1: Create new table with UUID as primary key (clean - no old_id or uuid columns)
@@ -149,10 +152,6 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
         # Build INSERT statement (only copy fields that exist in source)
         existing_fields = [f for f in fields_to_copy if f in values]
 
-        if i == 0:
-            print(f"[0029] Source columns: {col_names}")
-            print(f"[0029] Copying fields: {existing_fields}")
-
         placeholders = ", ".join(["?"] * (len(existing_fields) + 1))  # +1 for id
         field_list = "id, " + ", ".join(existing_fields)
 
@@ -164,13 +163,16 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
                 insert_values,
             )
             inserted_count += 1
+            if inserted_count % PROGRESS_EVERY == 0:
+                print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults...")
         except Exception as e:
             print(f"[0029] ERROR inserting record {old_id}: {e}")
             if i == 0:
                 print(f"[0029] First record values: {insert_values[:5]}...")
                 raise
 
-    print(f"[0029] Inserted {inserted_count}/{len(old_records)} records")
+    if old_records:
+        print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults")
 
     # Step 4: Replace old table with new table
     cursor.execute("DROP TABLE core_archiveresult")
@@ -185,7 +187,7 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
     cursor.execute("CREATE INDEX core_archiveresult_hook_name_idx ON core_archiveresult(hook_name)")
     cursor.execute("CREATE INDEX core_archiveresult_process_id_idx ON core_archiveresult(process_id)")
 
-    print(f"✓ Migrated {row_count} ArchiveResult records to UUID primary key")
+    print(f"    ✓ ArchiveResult UUID primary key migration complete ({row_count} records)")
 
 
 class Migration(migrations.Migration):

archivebox/core/migrations/0039_alter_archiveresult_process.py

@@ -0,0 +1,26 @@
+# Generated by Django 6.0.5 on 2026-05-27 20:11
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0038_snapshot_progress_idx"),
+        ("machine", "0015_process_progress_indexes"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="archiveresult",
+            name="process",
+            field=models.OneToOneField(
+                blank=True,
+                help_text="Process execution details for this archive result",
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="archiveresult",
+                to="machine.process",
+            ),
+        ),
+    ]

archivebox/core/migrations/0040_archiveresult_delete_at_snapshot_delete_at.py

@@ -0,0 +1,22 @@
+# Generated by Django 6.0.5 on 2026-05-27 20:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0039_alter_archiveresult_process"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="archiveresult",
+            name="delete_at",
+            field=models.DateTimeField(blank=True, db_index=True, default=None, null=True),
+        ),
+        migrations.AddField(
+            model_name="snapshot",
+            name="delete_at",
+            field=models.DateTimeField(blank=True, db_index=True, default=None, null=True),
+        ),
+    ]

archivebox/core/migrations/0041_snapshot_permissions.py

@@ -0,0 +1,23 @@
+# Generated by Django 6.0.5 on 2026-05-28 07:25
+
+import django.db.models.fields.json
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0040_archiveresult_delete_at_snapshot_delete_at"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="snapshot",
+            name="permissions",
+            field=models.GeneratedField(
+                db_index=True,
+                db_persist=True,
+                expression=django.db.models.fields.json.KeyTextTransform("PERMISSIONS", "config"),
+                output_field=models.CharField(max_length=16, null=True),
+            ),
+        ),
+    ]

archivebox/core/migrations/0042_snapshot_output_size.py

@@ -0,0 +1,43 @@
+# Generated by Django 6.0.5 on 2026-05-28 08:22
+
+from django.db import migrations, models
+from django.db.models import Sum
+
+
+def backfill_snapshot_output_size(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    ArchiveResult = apps.get_model("core", "ArchiveResult")
+    batch = []
+    rows = ArchiveResult.objects.values("snapshot_id").annotate(total_size=Sum("output_size")).order_by()
+    for row in rows.iterator(chunk_size=2000):
+        batch.append(Snapshot(id=row["snapshot_id"], output_size=row["total_size"] or 0))
+        if len(batch) >= 2000:
+            Snapshot.objects.bulk_update(batch, ["output_size"], batch_size=2000)
+            batch = []
+    if batch:
+        Snapshot.objects.bulk_update(batch, ["output_size"], batch_size=2000)
+
+
+def clear_snapshot_output_size(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    Snapshot.objects.update(output_size=0)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0041_snapshot_permissions"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="snapshot",
+            name="output_size",
+            field=models.BigIntegerField(
+                db_index=True,
+                default=0,
+                editable=False,
+                help_text="Total bytes of all ArchiveResult output files",
+            ),
+        ),
+        migrations.RunPython(backfill_snapshot_output_size, clear_snapshot_output_size),
+    ]

archivebox/core/migrations/0043_archiveresult_retry_at.py

@@ -0,0 +1,17 @@
+# Generated by Django 6.0.5 on 2026-05-28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0042_snapshot_output_size"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="archiveresult",
+            name="retry_at",
+            field=models.DateTimeField(blank=True, db_index=True, default=None, null=True),
+        ),
+    ]

archivebox/core/migrations/0044_alter_archiveresult_status_alter_snapshot_status.py

@@ -0,0 +1,41 @@
+# Generated by Django 6.0.5 on 2026-05-28 12:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0043_archiveresult_retry_at"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="archiveresult",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("queued", "Queued"),
+                    ("started", "Started"),
+                    ("paused", "Paused"),
+                    ("backoff", "Waiting to retry"),
+                    ("succeeded", "Succeeded"),
+                    ("failed", "Failed"),
+                    ("skipped", "Skipped"),
+                    ("noresults", "No Results"),
+                ],
+                db_index=True,
+                default="queued",
+                max_length=16,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="snapshot",
+            name="status",
+            field=models.CharField(
+                choices=[("queued", "Queued"), ("started", "Started"), ("paused", "Paused"), ("sealed", "Sealed")],
+                db_index=True,
+                default="queued",
+                max_length=15,
+            ),
+        ),
+    ]

archivebox/core/migrations/0045_archiveresult_unique_hook.py

@@ -0,0 +1,45 @@
+from django.db import migrations, models
+
+
+def deduplicate_archiveresults_per_hook(apps, schema_editor):
+    """Drop duplicate ArchiveResult rows per (snapshot, plugin, hook_name).
+
+    Real long-lived collections (cabbage's demo, beta-tester DBs) accumulated
+    multiple rows per hook over the dev rc chain. The next operation adds a
+    UniqueConstraint on that tuple; without this cleanup pass the constraint
+    fails with ``UNIQUE constraint failed`` mid-migration and bricks startup.
+    Keep the row with the highest id (most recent) for each tuple.
+    """
+    ArchiveResult = apps.get_model("core", "ArchiveResult")
+    duplicate_groups = (
+        ArchiveResult.objects.values("snapshot_id", "plugin", "hook_name").annotate(count=models.Count("id")).filter(count__gt=1)
+    )
+    for group in duplicate_groups.iterator(chunk_size=200):
+        lookup = {
+            "snapshot_id": group["snapshot_id"],
+            "plugin": group["plugin"],
+            "hook_name": group["hook_name"],
+        }
+        keep = ArchiveResult.objects.filter(**lookup).order_by("-id").values_list("id", flat=True).first()
+        if keep is not None:
+            ArchiveResult.objects.filter(**lookup).exclude(id=keep).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0044_alter_archiveresult_status_alter_snapshot_status"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            deduplicate_archiveresults_per_hook,
+            reverse_code=migrations.RunPython.noop,
+        ),
+        migrations.AddConstraint(
+            model_name="archiveresult",
+            constraint=models.UniqueConstraint(
+                fields=("snapshot", "plugin", "hook_name"),
+                name="unique_archiveresult_per_snapshot_hook",
+            ),
+        ),
+    ]

archivebox/core/migrations/0046_repair_snapshot_permissions.py

@@ -0,0 +1,43 @@
+from django.db import migrations
+
+
+def _repair_snapshot_permissions(apps, schema_editor):
+    """Backfill ``core_snapshot.permissions`` if a prior squash-skewed
+    ``django_migrations`` row claimed ``0041_snapshot_permissions`` was
+    applied but the column never actually landed on the table.
+
+    Beta-tester / cabbage-style DBs upgraded incrementally through the
+    0.8.x → 0.9.x rc chain have a 0041_snapshot_permissions entry with a
+    different historical effect (the name was reused across squashes),
+    so the runtime model's ``snapshot.permissions`` ``GeneratedField`` has
+    no underlying column. Fresh installs added the column via 0041 and
+    this is a no-op.
+    """
+    cursor = schema_editor.connection.cursor()
+    # ``table_xinfo`` lists STORED/VIRTUAL generated columns; ``table_info``
+    # silently drops them, so a prior 0041 that landed the STORED column
+    # would otherwise look absent here and we'd ALTER TABLE -> duplicate.
+    cursor.execute("PRAGMA table_xinfo(core_snapshot)")
+    existing_cols = {row[1] for row in cursor.fetchall()}
+    if "permissions" in existing_cols:
+        return
+    # See crawls/0016 — SQLite ALTER TABLE only allows VIRTUAL generated
+    # columns; STORED would error with "cannot add a STORED column". The
+    # runtime model's ``db_persist=True`` is only honored for fresh installs
+    # where 0041 added the column during initial table creation.
+    cursor.execute(
+        "ALTER TABLE core_snapshot ADD COLUMN permissions varchar(16) GENERATED ALWAYS AS (json_extract(config, '$.PERMISSIONS')) VIRTUAL",
+    )
+    cursor.execute(
+        "CREATE INDEX IF NOT EXISTS core_snapshot_permissions_idx ON core_snapshot (permissions)",
+    )
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0045_archiveresult_unique_hook"),
+    ]
+
+    operations = [
+        migrations.RunPython(_repair_snapshot_permissions, migrations.RunPython.noop),
+    ]

archivebox/core/migrations/0047_archiveresult_status_snapshot_index.py

@@ -0,0 +1,28 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0046_repair_snapshot_permissions"),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunSQL(
+                    sql="CREATE INDEX IF NOT EXISTS archiveresult_status_snap_idx ON core_archiveresult (status, snapshot_id)",
+                    reverse_sql="DROP INDEX IF EXISTS archiveresult_status_snap_idx",
+                ),
+            ],
+            state_operations=[
+                migrations.AddIndex(
+                    model_name="archiveresult",
+                    index=models.Index(fields=["status", "snapshot"], name="archiveresult_status_snap_idx"),
+                ),
+            ],
+        ),
+        migrations.RunSQL(
+            sql="ANALYZE core_archiveresult",
+            reverse_sql=migrations.RunSQL.noop,
+        ),
+    ]

archivebox/core/permissions.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from django.db.models import Q, QuerySet
+from django.http import HttpRequest
+
+PERMISSIONS_PUBLIC = "public"
+PERMISSIONS_UNLISTED = "unlisted"
+PERMISSIONS_PRIVATE = "private"
+PERMISSIONS_CHOICES = (
+    (PERMISSIONS_PUBLIC, "Public"),
+    (PERMISSIONS_UNLISTED, "Unlisted"),
+    (PERMISSIONS_PRIVATE, "Private"),
+)
+PERMISSIONS_VALUES = {value for value, _label in PERMISSIONS_CHOICES}
+PERMISSIONS_META = {
+    PERMISSIONS_PUBLIC: ("👥", "Public", "#047857", "#d1fae5"),
+    PERMISSIONS_UNLISTED: ("🔗", "Unlisted", "#1d4ed8", "#dbeafe"),
+    PERMISSIONS_PRIVATE: ("🔒", "Private", "#991b1b", "#fee2e2"),
+}
+
+
+def normalize_permissions(permissions: object, *, default: str = PERMISSIONS_PRIVATE) -> str:
+    permissions = str(permissions or "").strip().lower()
+    if permissions not in PERMISSIONS_VALUES:
+        return default
+    return permissions
+
+
+def is_admin_user(request: HttpRequest) -> bool:
+    user = request.user
+    return bool(user.is_authenticated and user.is_active and user.is_staff)
+
+
+def get_snapshot_permissions(snapshot) -> str:
+    permissions = snapshot.permissions
+    if not permissions:
+        permissions = snapshot.crawl.permissions
+    return normalize_permissions(permissions)
+
+
+def can_view_snapshot(request: HttpRequest, snapshot) -> bool:
+    permissions = get_snapshot_permissions(snapshot)
+    return permissions in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED} or is_admin_user(request)
+
+
+def _persona_ids_for_permissions(allowed_permissions: set[str]) -> list[str]:
+    from archivebox.personas.models import Persona
+
+    return [str(persona_id) for persona_id in Persona.objects.filter(permissions__in=allowed_permissions).values_list("id", flat=True)]
+
+
+def filter_personas_by_permissions(queryset: QuerySet, allowed_permissions: set[str]) -> QuerySet:
+    return queryset.filter(id__in=_persona_ids_for_permissions(allowed_permissions))
+
+
+def filter_snapshots_by_permissions(queryset: QuerySet, *, direct: bool = False, allowed_permissions: set[str] | None = None) -> QuerySet:
+    allowed_permissions = allowed_permissions or ({PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED} if direct else {PERMISSIONS_PUBLIC})
+    allowed = sorted(allowed_permissions)
+    return queryset.exclude(
+        (Q(permissions__isnull=False) & ~Q(permissions__in=allowed))
+        | (Q(permissions__isnull=True) & (Q(crawl__permissions__isnull=True) | ~Q(crawl__permissions__in=allowed))),
+    )
+
+
+def public_snapshots_queryset(queryset: QuerySet) -> QuerySet:
+    return filter_snapshots_by_permissions(queryset, direct=False)
+
+
+def direct_snapshots_queryset(request: HttpRequest, queryset: QuerySet) -> QuerySet:
+    return queryset if is_admin_user(request) else filter_snapshots_by_permissions(queryset, direct=True)

archivebox/core/recovery_util.py

@@ -0,0 +1,178 @@
+from __future__ import annotations
+
+from django.utils import timezone
+from rich.console import Console
+
+
+def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int]:
+    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.machine.models import Process
+    from django.db.models import Exists, OuterRef, Q, Subquery, Value
+    from django.db.models.functions import Coalesce
+
+    now = timezone.now()
+    recovery_console = Console(stderr=True, highlight=False, soft_wrap=True)
+    cleaned = {
+        "processes_stale_running": Process.cleanup_stale_running(),
+        "processes_orphaned_workers": Process.cleanup_orphaned_workers(),
+        "chrome_processes_orphaned": Process.cleanup_orphaned_chrome() if include_chrome else 0,
+        "crawls_queued_without_retry_at": 0,
+        "snapshots_queued_without_retry_at": 0,
+        "archiveresults_backoff": 0,
+        "archiveresults_started_without_running_process": 0,
+        "snapshots_started_without_running_results": 0,
+        "crawls_started_with_due_snapshots": 0,
+        "crawls_started_waiting_on_future_snapshots": 0,
+        "crawls_started_without_active_snapshots": 0,
+    }
+
+    running_archiveresults = ArchiveResult.objects.filter(
+        snapshot_id=OuterRef("pk"),
+        status=ArchiveResult.StatusChoices.STARTED,
+        process__status=Process.StatusChoices.RUNNING,
+    )
+    active_child_snapshots = Snapshot.objects.filter(
+        crawl_id=OuterRef("pk"),
+        status__in=Snapshot.OPEN_STATES,
+    )
+    due_child_snapshots = active_child_snapshots.exclude(status=Snapshot.StatusChoices.PAUSED).filter(
+        Q(retry_at__isnull=True) | Q(retry_at__lte=now),
+    )
+    next_future_child_retry = Subquery(
+        active_child_snapshots.filter(retry_at__gt=now).order_by("retry_at").values("retry_at")[:1],
+    )
+
+    # Broken lock repair: QUEUED rows with retry_at=NULL are invisible to the
+    # queue. Set only the scheduling field so the runner owns the next tick.
+    cleaned["crawls_queued_without_retry_at"] = Crawl.objects.filter(
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at__isnull=True,
+    ).update(retry_at=now, modified_at=now)
+    cleaned["snapshots_queued_without_retry_at"] = Snapshot.objects.filter(
+        status=Snapshot.StatusChoices.QUEUED,
+        retry_at__isnull=True,
+        crawl__status__in=Crawl.RUNNABLE_STATES,
+    ).update(retry_at=now, modified_at=now)
+    backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF)
+    orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED).exclude(
+        process__status=Process.StatusChoices.RUNNING,
+    )
+    # ArchiveResult has no retry_at scheduler. Wake only the parent Snapshots
+    # for result rows we are about to repair, then requeue those exact rows.
+    # Using subqueries keeps million-row recovery in SQLite instead of building
+    # Python ID lists or scanning all sealed snapshots.
+    Snapshot.objects.filter(
+        id__in=backoff_results.values("snapshot_id"),
+        status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
+        retry_at__isnull=True,
+    ).update(retry_at=now, modified_at=now)
+    Snapshot.objects.filter(
+        id__in=orphaned_results.values("snapshot_id"),
+        status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
+        retry_at__isnull=True,
+    ).update(retry_at=now, modified_at=now)
+    cleaned["archiveresults_backoff"] = backoff_results.update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now)
+    # Impossible state repair: STARTED ArchiveResults without a live Process
+    # have no owner left to emit completion. Requeue only the result row; the
+    # snapshot/crawl schedulers will pick up normal retry processing.
+    cleaned["archiveresults_started_without_running_process"] = orphaned_results.update(
+        status=ArchiveResult.StatusChoices.QUEUED,
+        process=None,
+        modified_at=now,
+    )
+    started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter(
+        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+    )
+
+    # Broken lock repair: STARTED + retry_at=NULL or retry_at in the future
+    # means "owned by an active runner". Recovery only runs from the current
+    # elected runner after Process cleanup has proven old owners are gone, so
+    # STARTED rows with no live ArchiveResult process should not wait out the
+    # previous runner's full lease before the new runner can resume them.
+    # We only unlock scheduling; normal Snapshot runner code owns the next
+    # transition and side effects.
+    cleaned["snapshots_started_without_running_results"] = (
+        started_snapshots.annotate(has_running_results=Exists(running_archiveresults))
+        .filter(has_running_results=False)
+        .update(
+            retry_at=now,
+            modified_at=now,
+        )
+    )
+
+    # Broken lock repair: STARTED + retry_at=NULL is an orphaned ownership
+    # lease. Recovery only unlocks scheduling; the runner owns any subsequent
+    # state-machine transition, including sealing rows whose children/results
+    # are already final.
+    recoverable_started_crawls = Crawl.objects.filter(status=Crawl.StatusChoices.STARTED).filter(
+        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+    )
+
+    due_started_crawls = recoverable_started_crawls.annotate(has_due_child=Exists(due_child_snapshots)).filter(has_due_child=True)
+    cleaned["crawls_started_with_due_snapshots"] = due_started_crawls.update(retry_at=now, modified_at=now)
+    future_started_crawls = recoverable_started_crawls.annotate(
+        has_active_child=Exists(active_child_snapshots),
+        has_due_child=Exists(due_child_snapshots),
+        next_child_retry=next_future_child_retry,
+    ).filter(has_active_child=True, has_due_child=False)
+    cleaned["crawls_started_waiting_on_future_snapshots"] = future_started_crawls.update(
+        retry_at=Coalesce("next_child_retry", Value(now)),
+        modified_at=now,
+    )
+    finished_started_crawls = recoverable_started_crawls.annotate(has_active_child=Exists(active_child_snapshots)).filter(
+        has_active_child=False,
+    )
+    cleaned["crawls_started_without_active_snapshots"] = finished_started_crawls.update(retry_at=now, modified_at=now)
+
+    repair_messages = {
+        "processes_stale_running": (
+            "Closing {count} interrupted process(es) "
+            "(ArchiveBox may have been interrupted before it was able to record that they stopped; any affected work can now be retried)."
+        ),
+        "processes_orphaned_workers": (
+            "Closing {count} interrupted extractor process(es) "
+            "(ArchiveBox may have been interrupted before it was able to record their result; affected extractor results can now be retried)."
+        ),
+        "chrome_processes_orphaned": (
+            "Stopping {count} leftover browser process(es) "
+            "(ArchiveBox may have been interrupted before it was able to close them; this frees browser resources and avoids duplicate browser sessions)."
+        ),
+        "crawls_queued_without_retry_at": (
+            "Starting {count} Crawl(s) that were queued but never started "
+            "(ArchiveBox may have been interrupted before it was able to begin archiving them)."
+        ),
+        "snapshots_queued_without_retry_at": (
+            "Starting {count} Snapshot(s) that were queued but never started "
+            "(ArchiveBox may have been interrupted before it was able to archive those URLs)."
+        ),
+        "archiveresults_backoff": (
+            "Retrying {count} extractor result(s) that were waiting to retry "
+            "(ArchiveBox may have been interrupted before it was able to try them again; affected outputs will be retried)."
+        ),
+        "archiveresults_started_without_running_process": (
+            "Retrying {count} extractor result(s) that were interrupted before finishing "
+            "(ArchiveBox may have been interrupted before it was able to save their final status; partial files will be overwritten with fresh results upon retry)."
+        ),
+        "snapshots_started_without_running_results": (
+            "Resuming {count} Snapshot(s) that were interrupted before finishing "
+            "(ArchiveBox may have been interrupted before it was able to finish archiving them; missing outputs will be retried)."
+        ),
+        "crawls_started_with_due_snapshots": (
+            "Resuming {count} Crawl(s) with pending URLs ready to archive "
+            "(ArchiveBox may have been interrupted before it was able to archive the remaining URLs; pending URLs will continue)."
+        ),
+        "crawls_started_waiting_on_future_snapshots": (
+            "Resuming {count} Crawl(s) with URLs waiting for a later retry "
+            "(ArchiveBox may have been interrupted before it was able to retry delayed URLs; they will retry later)."
+        ),
+        "crawls_started_without_active_snapshots": (
+            "Finalizing {count} Crawl(s) that finished URL processing but were not closed cleanly "
+            "(ArchiveBox may have been interrupted before it was able to save the final crawl status; archived data is not changed)."
+        ),
+    }
+    for key, message in repair_messages.items():
+        if cleaned[key]:
+            recovery_console.print(f"[yellow]⚠️ Repairing: {message.format(count=cleaned[key])}[/yellow]")
+
+    return cleaned

archivebox/core/routes_util.py

@@ -0,0 +1,381 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+from archivebox.config.common import get_config
+
+
+_SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
+_SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P<suffix>[0-9a-fA-F]{12})$")
+_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api", "public")
+
+
+def split_host_port(host: str) -> tuple[str, str | None]:
+    parsed = urlparse(f"//{host}")
+    hostname = (parsed.hostname or host or "").lower()
+    port = str(parsed.port) if parsed.port else None
+    return hostname, port
+
+
+def _normalize_base_url(value: str | None) -> str:
+    if not value:
+        return ""
+    base = value.strip()
+    if not base:
+        return ""
+    if "://" not in base:
+        base = f"http://{base}"
+    parsed = urlparse(base)
+    if not parsed.netloc:
+        return ""
+    # Accept ``*.<host>`` as a synonym for ``<host>`` so users can paste the
+    # wildcard-friendly form (e.g. from the banner suggestion) without it
+    # leaking ``*.`` into every downstream URL. Subdomain routing already
+    # prepends the appropriate role label (admin/web/api/snap-*) at build
+    # time, so the bare base host is what we want to store.
+    netloc = parsed.netloc
+    while netloc.startswith("*."):
+        netloc = netloc[2:]
+    if not netloc:
+        return ""
+    return f"{parsed.scheme}://{netloc}"
+
+
+def normalize_base_url(value: str | None) -> str:
+    return _normalize_base_url(value)
+
+
+def _csrf_trusted_origins(config) -> list[str]:
+    raw = (config.CSRF_TRUSTED_ORIGINS or "").strip()
+    if not raw:
+        return []
+    seen: list[str] = []
+    for entry in raw.split(","):
+        normalized = _normalize_base_url(entry.strip())
+        if normalized and normalized not in seen:
+            seen.append(normalized)
+    return seen
+
+
+def _allowed_hosts(config) -> set[str]:
+    raw = (config.ALLOWED_HOSTS or "").strip()
+    if not raw:
+        return set()
+    return {entry.strip().lower() for entry in raw.split(",") if entry.strip() and entry.strip() != "*"}
+
+
+def derive_base_url_from_csrf(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    """Pick a single CSRF_TRUSTED_ORIGINS entry to act as the implicit BASE_URL.
+
+    0.7.3 → 0.9.0 upgrade path: any reverse-proxied 0.7.3 deployment already
+    had ``CSRF_TRUSTED_ORIGINS=https://archive.example.com`` set (required for
+    admin login to work). On upgrade, ``BASE_URL`` is the new knob — but it
+    defaults to empty, and falling through to ``BIND_ADDR`` produces an
+    unreachable URL like ``http://0.0.0.0:8000``. If the user has exactly one
+    CSRF origin we treat it as the implicit BASE_URL so links/redirects keep
+    pointing at the public hostname they already configured.
+
+    Returns ``""`` when the inference is ambiguous (multiple origins) or
+    impossible (none set) so callers fall through to their next strategy.
+    """
+    config = config or get_config(**config_kwargs)
+    origins = _csrf_trusted_origins(config)
+    if len(origins) == 1:
+        return origins[0]
+    return ""
+
+
+def get_listen_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    return (config.BIND_ADDR or "").strip()
+
+
+def get_listen_parts(config: dict[str, Any] | None = None, **config_kwargs: Any) -> tuple[str, str | None]:
+    config = config or get_config(**config_kwargs)
+    return split_host_port(get_listen_host(config=config))
+
+
+def _with_port(host: str, port: str | None) -> str:
+    return f"{host}:{port}" if port else host
+
+
+def strip_role_subdomain(host: str) -> str:
+    """Strip leading ``admin.`` / ``web.`` / ``api.`` / ``public.`` / ``snap-*.``
+    labels from a host (preserving the port). Strips repeatedly so an
+    already-compounded host like ``snap-X.snap-X.<base>`` reduces all the
+    way down to ``<base>``.
+
+    Used when we want to recover the canonical base host from a request that
+    arrived on a role subdomain — otherwise builders that prepend their own
+    role label (e.g. ``snap-X.``) compound onto the existing prefix and you
+    get ``snap-X.snap-X.snap-X.<base>`` on every click.
+    """
+    if not host:
+        return ""
+    hostname, port = split_host_port(host)
+    while hostname and "." in hostname:
+        head, _sep, rest = hostname.partition(".")
+        if head in _ROLE_SUBDOMAIN_LABELS or _SNAPSHOT_SUBDOMAIN_RE.match(head):
+            hostname = rest
+            continue
+        break
+    return _with_port(hostname, port)
+
+
+def _is_local_bind_host(host: str) -> bool:
+    return host in {"", "0.0.0.0", "::", "127.0.0.1", "::1", "localhost"}
+
+
+def canonical_base_host_for_request(request_host: str) -> str:
+    """Strip role subdomains and remap loopback hostnames to ``archivebox.localhost``.
+
+    Used by the banner suggestion and the in-browser pin endpoint: when the
+    user is hitting the server on raw ``localhost:9292`` or ``127.0.0.1:9292``
+    we want to suggest the wildcard-friendly ``archivebox.localhost`` family
+    instead, so the eventual pinned ``BASE_URL`` plays nicely with subdomain
+    routing without forcing the user to add a /etc/hosts entry.
+    """
+    hostname, port = split_host_port(strip_role_subdomain(request_host or ""))
+    if _is_local_bind_host(hostname):
+        hostname = "archivebox.localhost"
+    return _with_port(hostname, port)
+
+
+def _root_host_from_listen(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    listen_host, listen_port = get_listen_parts(config=config)
+    root_host = "archivebox.localhost" if _is_local_bind_host(listen_host) else listen_host
+    return _with_port(root_host, listen_port) if root_host else ""
+
+
+def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    override = _normalize_base_url(config.BASE_URL)
+    if override:
+        return override
+
+    # A) Implicit BASE_URL from a single CSRF_TRUSTED_ORIGINS entry. Catches
+    # 0.7.3 → 0.9.0 upgrades where users already set CSRF_TRUSTED_ORIGINS
+    # for their reverse-proxy login but never set BASE_URL.
+    csrf_derived = derive_base_url_from_csrf(config)
+    if csrf_derived:
+        return csrf_derived
+
+    scheme = request.scheme if request else "http"
+    if request:
+        req_host, req_port = split_host_port(request.get_host())
+        if req_host.endswith(".archivebox.localhost"):
+            return f"{scheme}://{_with_port('archivebox.localhost', req_port)}"
+        if _is_local_bind_host(req_host):
+            return f"{scheme}://{_with_port('archivebox.localhost', req_port)}"
+        # C) Per-request fallback: when ``BASE_URL`` is unset and CSRF didn't
+        # give us a single origin, trust the request's Host header — but first
+        # peel off any ``admin.`` / ``web.`` / ``api.`` / ``public.`` /
+        # ``snap-*.`` label. Otherwise the URL builders below prepend their own
+        # role label onto a host that already carries one, producing the
+        # ``snap-X.snap-X.snap-X.<base>`` compounding bug. Django has already
+        # admitted the host via ALLOWED_HOSTS; the misconfig banner surfaces
+        # the case where the resulting URL doesn't match what the operator
+        # probably intended.
+        canonical_host = strip_role_subdomain(request.get_host())
+        return f"{scheme}://{canonical_host}"
+
+    root_host = _root_host_from_listen(config=config)
+    return f"{scheme}://{root_host}" if root_host else ""
+
+
+def get_base_host(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    return urlparse(get_base_url(request=request, config=config, **config_kwargs)).netloc.lower()
+
+
+def _build_base_host(subdomain: str | None, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    base_host = get_base_host(request=request, config=config, **config_kwargs)
+    if not base_host:
+        return ""
+    host, port = split_host_port(base_host)
+    full_host = f"{subdomain}.{host}" if subdomain else host
+    return _with_port(full_host, port)
+
+
+def get_admin_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return get_base_host(config=config)
+    return _build_base_host("admin", config=config)
+
+
+def get_web_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return get_base_host(config=config)
+    return _build_base_host("web", config=config)
+
+
+def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return get_base_host(config=config)
+    return _build_base_host("api", config=config)
+
+
+def get_public_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return get_base_host(config=config)
+    return _build_base_host("public", config=config)
+
+
+def get_snapshot_subdomain(snapshot_id: str) -> str:
+    normalized = re.sub(r"[^0-9a-fA-F]", "", snapshot_id or "")
+    suffix = (normalized[-12:] if len(normalized) >= 12 else normalized).lower()
+    return f"snap-{suffix}"
+
+
+def get_snapshot_host(snapshot_id: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return get_base_host(config=config)
+    return _build_base_host(get_snapshot_subdomain(snapshot_id), config=config)
+
+
+def get_original_host(domain: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return get_base_host(config=config)
+    return _build_base_host(domain, config=config)
+
+
+def is_snapshot_subdomain(subdomain: str) -> bool:
+    value = (subdomain or "").strip()
+    return bool(_SNAPSHOT_SUBDOMAIN_RE.match(value) or _SNAPSHOT_ID_RE.match(value))
+
+
+def get_snapshot_lookup_key(snapshot_ref: str) -> str:
+    value = (snapshot_ref or "").strip().lower()
+    match = _SNAPSHOT_SUBDOMAIN_RE.match(value)
+    if match:
+        return match.group("suffix")
+    if _SNAPSHOT_ID_RE.match(value):
+        return re.sub(r"[^0-9a-fA-F]", "", value).lower()
+    return value
+
+
+def get_listen_subdomain(request_host: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return ""
+    req_host, req_port = split_host_port(request_host)
+    base_host, base_port = split_host_port(get_base_host(config=config))
+    if not base_host:
+        return ""
+    if base_port and req_port and base_port != req_port:
+        return ""
+    if req_host == base_host:
+        return ""
+    suffix = f".{base_host}"
+    if req_host.endswith(suffix):
+        return req_host[: -len(suffix)]
+    return ""
+
+
+def host_matches(request_host: str, target_host: str) -> bool:
+    if not request_host or not target_host:
+        return False
+    req_host, req_port = split_host_port(request_host)
+    target_host_only, target_port = split_host_port(target_host)
+    if req_host != target_host_only:
+        return False
+    if target_port and req_port and target_port != req_port:
+        return False
+    return True
+
+
+def _scheme_from_request(request=None, config: dict[str, Any] | None = None) -> str:
+    config = config or get_config()
+    override = _normalize_base_url(config.BASE_URL)
+    if override:
+        return urlparse(override).scheme
+    if request:
+        return request.scheme
+    return "http"
+
+
+def _build_base_url_for_host(host: str, request=None, config: dict[str, Any] | None = None) -> str:
+    if not host:
+        return ""
+    scheme = _scheme_from_request(request, config=config)
+    return f"{scheme}://{host}"
+
+
+def get_admin_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return get_base_url(request=request, config=config)
+    return _build_base_url_for_host(_build_base_host("admin", request=request, config=config), request=request, config=config)
+
+
+def get_web_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return get_base_url(request=request, config=config)
+    return _build_base_url_for_host(_build_base_host("web", request=request, config=config), request=request, config=config)
+
+
+def get_api_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return get_base_url(request=request, config=config)
+    return _build_base_url_for_host(_build_base_host("api", request=request, config=config), request=request, config=config)
+
+
+def get_public_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return get_base_url(request=request, config=config)
+    return _build_base_url_for_host(_build_base_host("public", request=request, config=config), request=request, config=config)
+
+
+def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return _build_url(get_web_base_url(request=request, config=config), f"/snapshot/{str(snapshot_id).replace('-', '')}")
+    return _build_base_url_for_host(
+        _build_base_host(get_snapshot_subdomain(snapshot_id), request=request, config=config),
+        request=request,
+        config=config,
+    )
+
+
+def get_original_base_url(domain: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return _build_url(get_web_base_url(request=request, config=config), f"/original/{domain}")
+    return _build_base_url_for_host(_build_base_host(domain, request=request, config=config), request=request, config=config)
+
+
+def build_admin_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    return _build_url(get_admin_base_url(request, config=config, **config_kwargs), path)
+
+
+def build_web_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    return _build_url(get_web_base_url(request, config=config, **config_kwargs), path)
+
+
+def build_snapshot_url(snapshot_id: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    return _build_url(get_snapshot_base_url(snapshot_id, request=request, config=config, **config_kwargs), path)
+
+
+def build_original_url(domain: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    return _build_url(get_original_base_url(domain, request=request, config=config, **config_kwargs), path)
+
+
+def _build_url(base_url: str, path: str) -> str:
+    if not base_url:
+        if not path:
+            return ""
+        return path if path.startswith("/") else f"/{path}"
+    if not path:
+        return base_url
+    return f"{base_url}{path if path.startswith('/') else f'/{path}'}"

archivebox/core/settings.py

@@ -8,13 +8,12 @@ import importlib
 from pathlib import Path
 
 from django.conf.locale.en import formats as en_formats  # type: ignore
-from django.utils.crypto import get_random_string
 
 import archivebox
 
 from archivebox.config.constants import CONSTANTS
 from archivebox.config.common import get_config
-from archivebox.core.host_utils import normalize_base_url, get_admin_base_url, get_api_base_url
+from archivebox.core.routes_util import normalize_base_url, get_admin_base_url, get_api_base_url
 from .settings_logging import SETTINGS_LOGGING
 
 
@@ -41,7 +40,7 @@ ASGI_APPLICATION = "archivebox.core.asgi.application"
 ROOT_URLCONF = "archivebox.core.urls"
 
 LOGIN_URL = "/accounts/login/"
-LOGOUT_REDIRECT_URL = os.environ.get("LOGOUT_REDIRECT_URL", "/")
+LOGOUT_REDIRECT_URL = CONFIG.LOGOUT_REDIRECT_URL
 
 PASSWORD_RESET_URL = "/accounts/password_reset/"
 APPEND_SLASH = True
@@ -64,14 +63,15 @@ INSTALLED_APPS = [
     # Our ArchiveBox-provided apps (use fully qualified names)
     # NOTE: Order matters! Apps with migrations that depend on other apps must come AFTER their dependencies
     # "archivebox.config",  # ArchiveBox config settings (no models, not a real Django app)
+    "archivebox.plugins",  # plugin discovery, hook helpers, config UI, and plugin metadata views
+    "archivebox.search",  # search backend query helpers, admin search UI, and daemon integrations
     "archivebox.machine",  # handles collecting and storing information about the host machine, network interfaces, binaries, etc.
     "archivebox.workers",  # handles starting and managing background workers and processes (orchestrators and actors)
     "archivebox.personas",  # handles Persona and session management
     "archivebox.core",  # core django model with Snapshot, ArchiveResult, etc. (crawls depends on this)
     "archivebox.crawls",  # handles Crawl and CrawlSchedule models and management (depends on core)
+    "archivebox.progressmonitor",  # live progress endpoint and admin monitor template
     "archivebox.api",  # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
-    # ArchiveBox plugins (hook-based plugins no longer add Django apps)
-    # Use hooks.py discover_hooks() for plugin functionality
     # 3rd-party apps from PyPI that need to be loaded last
     "admin_data_views",  # handles rendering some convenient automatic read-only views of data in Django admin
     "django_extensions",  # provides Django Debug Toolbar (and other non-debug helpers)
@@ -210,6 +210,7 @@ TEMPLATES = [
                 "django.template.context_processors.request",
                 "django.contrib.auth.context_processors.auth",
                 "django.contrib.messages.context_processors.messages",
+                "archivebox.core.context_processors.archivebox_globals",
             ],
         },
     },
@@ -224,23 +225,29 @@ TEMPLATES = [
 # CACHE_DB_PATH = CONSTANTS.CACHE_DIR / CACHE_DB_FILENAME
 # CACHE_DB_TABLE = 'django_cache'
 
-DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", str(CONSTANTS.DATABASE_FILE))
-SQLITE_JOURNAL_MODE = os.environ.get("ARCHIVEBOX_SQLITE_JOURNAL_MODE", "WAL")
-SQLITE_MMAP_SIZE = os.environ.get("ARCHIVEBOX_SQLITE_MMAP_SIZE", "0" if CONSTANTS.IN_DOCKER else "134217728")
+DATABASE_NAME = CONFIG.DATABASE_NAME
+SQLITE_JOURNAL_MODE = CONFIG.SQLITE_JOURNAL_MODE
+SQLITE_MMAP_SIZE = CONFIG.SQLITE_MMAP_SIZE
 
 SQLITE_CONNECTION_OPTIONS = {
-    "ENGINE": "django.db.backends.sqlite3",
+    "ENGINE": "archivebox.core.sqlite_backend",
     "TIME_ZONE": CONSTANTS.TIMEZONE,
     "OPTIONS": {
         # https://gcollazo.com/optimal-sqlite-settings-for-django/
         # https://litestream.io/tips/#busy-timeout
         # https://docs.djangoproject.com/en/5.1/ref/databases/#setting-pragma-options
-        "timeout": 30,
+        "timeout": CONFIG.SQLITE_BUSY_TIMEOUT / 1000,
         "check_same_thread": False,
-        "transaction_mode": "IMMEDIATE",
+        # Keep SQLite on Django's default deferred transaction mode. BEGIN
+        # IMMEDIATE grabs the write lock as soon as atomic() opens, which is
+        # exactly what hurts ArchiveBox on large collections where Python code
+        # may do filesystem work before the actual row write. Deferred BEGIN
+        # keeps writes statement-scoped unless a caller explicitly opens a
+        # transaction around multiple writes.
+        "transaction_mode": None,
         "init_command": (
             "PRAGMA foreign_keys=ON;"
-            "PRAGMA busy_timeout = 30000;"
+            f"PRAGMA busy_timeout = {CONFIG.SQLITE_BUSY_TIMEOUT};"
             f"PRAGMA journal_mode = {SQLITE_JOURNAL_MODE};"
             "PRAGMA synchronous = NORMAL;"
             "PRAGMA temp_store = MEMORY;"
@@ -269,7 +276,7 @@ MIGRATION_MODULES = {"signal_webhooks": None}
 
 # Django requires DEFAULT_AUTO_FIELD to subclass AutoField (BigAutoField, SmallAutoField, etc.)
 # Cannot use UUIDField here until Django 6.0 introduces DEFAULT_PK_FIELD setting
-# For now: manually add `id = models.UUIDField(primary_key=True, default=uuid7, ...)` to all models
+# For now: manually add `id = CompactUUIDField(primary_key=True, default=uuid7, ...)` to all models
 # OR inherit from ModelWithUUID base class which provides UUID primary key
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 
@@ -352,10 +359,37 @@ CHANNEL_LAYERS = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"}
 ### Security Settings
 ################################################################################
 
-SECRET_KEY = CONFIG.SECRET_KEY or get_random_string(50, "abcdefghijklmnopqrstuvwxyz0123456789_")
+# Persist SECRET_KEY on first use. Data dirs created before init wrote a
+# SECRET_KEY line — or those whose ArchiveBox.conf was hand-edited to remove
+# it — would otherwise sign session cookies with a fresh random key on every
+# boot (because the pydantic field's default_factory regenerates), logging
+# users out on every server restart. ``archivebox init`` writes this for new
+# collections; this branch is the recovery path for the rest.
+#
+# We can't check ``CONFIG.SECRET_KEY`` for "missing" — pydantic's
+# default_factory already filled it with a fresh random value. We have to
+# inspect the conf file directly to know whether the value will survive.
+SECRET_KEY = CONFIG.SECRET_KEY
+try:
+    from archivebox.config.configset import BaseConfigSet as _BaseConfigSet
 
-ALLOWED_HOSTS = CONFIG.ALLOWED_HOSTS.split(",")
-CSRF_TRUSTED_ORIGINS = list(set(CONFIG.CSRF_TRUSTED_ORIGINS.split(",")))
+    _persisted_keys = _BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
+    _secret_persisted = bool((_persisted_keys.get("SECRET_KEY") or "").strip())
+except Exception:
+    _secret_persisted = True  # err on the side of NOT touching disk
+if not _secret_persisted:
+    try:
+        from archivebox.config.collection import write_config_file
+
+        write_config_file({"SECRET_KEY": SECRET_KEY})
+    except Exception:
+        # Read-only mount, missing data dir, mid-init race — fall back to the
+        # in-memory random key. The user will get logged out on the next boot
+        # but the server still comes up.
+        pass
+
+ALLOWED_HOSTS = [host.strip() for host in CONFIG.ALLOWED_HOSTS.split(",") if host.strip()]
+CSRF_TRUSTED_ORIGINS = list({origin.strip() for origin in CONFIG.CSRF_TRUSTED_ORIGINS.split(",") if origin.strip()})
 
 admin_base_url = normalize_base_url(get_admin_base_url())
 if admin_base_url and admin_base_url not in CSRF_TRUSTED_ORIGINS:
@@ -365,12 +399,17 @@ api_base_url = normalize_base_url(get_api_base_url())
 if api_base_url and api_base_url not in CSRF_TRUSTED_ORIGINS:
     CSRF_TRUSTED_ORIGINS.append(api_base_url)
 
-# automatically fix case when user sets ALLOWED_HOSTS (e.g. to archivebox.example.com)
-# but forgets to add https://archivebox.example.com to CSRF_TRUSTED_ORIGINS
+# Auto-extend CSRF_TRUSTED_ORIGINS with the explicit ALLOWED_HOSTS entries so
+# users who set ALLOWED_HOSTS=archivebox.example.com don't have to also list
+# https://archivebox.example.com under CSRF_TRUSTED_ORIGINS. (The previous
+# per-host WARNING print was just noise — the auto-append below is the actual
+# fix, and the effective CSRF_TRUSTED_ORIGINS gets surfaced once at startup
+# from archivebox_server.py.)
 for hostname in ALLOWED_HOSTS:
+    if hostname == "*":
+        continue
     https_endpoint = f"https://{hostname}"
-    if hostname != "*" and https_endpoint not in CSRF_TRUSTED_ORIGINS:
-        print(f"[!] WARNING: {https_endpoint} from ALLOWED_HOSTS should be added to CSRF_TRUSTED_ORIGINS")
+    if https_endpoint not in CSRF_TRUSTED_ORIGINS:
         CSRF_TRUSTED_ORIGINS.append(https_endpoint)
 
 SECURE_BROWSER_XSS_FILTER = True
@@ -380,6 +419,14 @@ SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
 CSRF_COOKIE_SECURE = False
 SESSION_COOKIE_SECURE = False
 SESSION_COOKIE_HTTPONLY = True
+SESSION_COOKIE_NAME = f"archivebox_sessionid_{CONSTANTS.COLLECTION_ID}"
+CSRF_COOKIE_NAME = f"archivebox_csrftoken_{CONSTANTS.COLLECTION_ID}"
+# Auth cookies are intentionally scoped to the exact host that set them so
+# the admin session is NOT readable from public.* / web.* / api.* — that
+# split is a security boundary, not a UX choice. Subdomains that need to
+# render an "is the user logged in?" indicator must use the single-bit
+# `archivebox_admin_logged_in` hint cookie set by core/middleware.py
+# (which IS scoped to the listen-host parent), never widen these.
 SESSION_COOKIE_DOMAIN = None
 CSRF_COOKIE_DOMAIN = None
 SESSION_COOKIE_AGE = 1209600  # 2 weeks
@@ -486,11 +533,11 @@ ADMIN_DATA_VIEWS = {
         },
         {
             "route": "plugins/",
-            "view": "archivebox.config.views.plugins_list_view",
+            "view": "archivebox.plugins.views.plugins_list_view",
             "name": "Plugins",
             "items": {
                 "route": "<str:key>/",
-                "view": "archivebox.config.views.plugin_detail_view",
+                "view": "archivebox.plugins.views.plugin_detail_view",
                 "name": "plugin",
             },
         },

archivebox/core/settings_logging.py

@@ -7,6 +7,7 @@ import logging
 
 
 from archivebox.config import CONSTANTS
+from archivebox.misc.logging import STDERR
 
 
 IGNORABLE_URL_PATTERNS = [
@@ -42,6 +43,65 @@ class NoisyRequestsFilter(logging.Filter):
         return True
 
 
+class DaphneCloseTimeoutFilter(logging.Filter):
+    """Drop daphne's noisy "killed slow response after client disconnect" warning.
+
+    Daphne emits this whenever a request handler is still running when the
+    client disconnects (e.g. iframe gets navigated away mid-response while
+    fetching favicon / screenshot / preview html). For our use case these are
+    always benign — the disconnect is the browser cancelling a request, not a
+    server-side fault — so we suppress them outright rather than spamming
+    WARNING. Other daphne.server lines pass through unchanged.
+    """
+
+    def filter(self, record) -> bool:
+        if record.name != "daphne.server":
+            return True
+        logline = record.getMessage()
+        if (
+            "Application instance" in logline
+            and "for connection <WebRequest" in logline
+            and "took too long to shut down" in logline
+            and "was killed" in logline
+        ):
+            return False
+        return True
+
+
+class AsyncioCancelledShieldFilter(logging.Filter):
+    """Drop asyncio's "CancelledError exception in shielded future" noise.
+
+    When a browser disconnects mid-request, daphne cancels the asgi task and
+    asgiref's ``sync_to_async`` shields the synchronous Django view via
+    ``asyncio.shield(exec_coro)``. The shield wakes up to find its parent
+    cancelled and re-raises ``CancelledError``; asyncio's default exception
+    handler then logs the full traceback via the ``asyncio`` logger. There's
+    nothing the server can do (the client is already gone), so these tracebacks
+    are pure noise and cause hundreds of lines of spam per disconnect.
+
+    We match conservatively: only drop the specific shielded-future message
+    that points back into asgiref's shield path. Other asyncio errors fall
+    through unchanged.
+    """
+
+    def filter(self, record) -> bool:
+        if record.name != "asyncio":
+            return True
+        logline = record.getMessage()
+        if "CancelledError exception in shielded future" in logline:
+            return False
+        exc_info = record.exc_info
+        if exc_info and exc_info[0] is not None:
+            try:
+                import asyncio as _asyncio
+
+                if issubclass(exc_info[0], _asyncio.CancelledError):
+                    return False
+            except Exception:
+                pass
+        return True
+
+
 class CustomOutboundWebhookLogFormatter(logging.Formatter):
     def format(self, record):
         result = super().format(record)
@@ -100,6 +160,12 @@ SETTINGS_LOGGING = {
         "noisyrequestsfilter": {
             "()": NoisyRequestsFilter,
         },
+        "daphneclosetimeout": {
+            "()": DaphneCloseTimeoutFilter,
+        },
+        "asynciocancelledshield": {
+            "()": AsyncioCancelledShieldFilter,
+        },
         "stripansi": {
             "()": StripANSIColorCodesFilter,
         },
@@ -117,7 +183,8 @@ SETTINGS_LOGGING = {
             "level": "DEBUG",
             "markup": False,
             "rich_tracebacks": False,  # Use standard Python tracebacks (no frame/box)
-            "filters": ["noisyrequestsfilter", "stripansi"],
+            "console": STDERR,
+            "filters": ["noisyrequestsfilter", "daphneclosetimeout", "asynciocancelledshield", "stripansi"],
         },
         "logfile": {
             "level": "INFO",
@@ -126,7 +193,7 @@ SETTINGS_LOGGING = {
             "maxBytes": 1024 * 1024 * 25,  # 25 MB
             "backupCount": 10,
             "formatter": "rich",
-            "filters": ["noisyrequestsfilter", "stripansi"],
+            "filters": ["noisyrequestsfilter", "daphneclosetimeout", "asynciocancelledshield", "stripansi"],
         },
         "outbound_webhooks": {
             "class": "rich.logging.RichHandler",

archivebox/core/shutdown_util.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+import os
+import signal
+import subprocess
+import sys
+import threading
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+
+import psutil
+
+
+@dataclass
+class ShutdownSignalState:
+    """Tracks the exact OS signal that asked a foreground command to exit."""
+
+    signal_name: str | None = None
+
+
+_active_shutdown_state: ShutdownSignalState | None = None
+
+
+def raise_if_shutdown_requested() -> None:
+    """Let long foreground loops honor a signal even if Python ignored it once."""
+
+    if _active_shutdown_state and _active_shutdown_state.signal_name:
+        raise KeyboardInterrupt
+
+
+def configured_stopwaitsecs(workers: list[dict[str, str]] | tuple[dict[str, str], ...], *, default: int = 5, buffer: int = 5) -> int:
+    """Return a deterministic shutdown bound from generated worker definitions."""
+
+    stop_grace_seconds = default
+    for worker in workers:
+        try:
+            stop_grace_seconds = max(stop_grace_seconds, int(worker.get("stopwaitsecs") or default) + buffer)
+        except (TypeError, ValueError):
+            continue
+    return stop_grace_seconds
+
+
+def wait_popen_and_kill_children(
+    proc: subprocess.Popen,
+    children: list[psutil.Process],
+    *,
+    timeout: float,
+    kill_timeout: float = 2.0,
+) -> None:
+    """Wait for a Popen parent and then hard-kill any surviving descendants."""
+
+    try:
+        proc.wait(timeout=timeout)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+        proc.wait(timeout=kill_timeout)
+    kill_remaining_processes(children, timeout=kill_timeout)
+
+
+def wait_psutil_and_kill_children(
+    proc: psutil.Process,
+    children: list[psutil.Process],
+    *,
+    timeout: float,
+    kill_timeout: float = 2.0,
+) -> None:
+    """Wait for a psutil parent and then hard-kill any surviving descendants."""
+
+    try:
+        if proc.status() == psutil.STATUS_ZOMBIE:
+            # Another ArchiveBox foreground parent owns this Popen and must reap
+            # it. By the time supervisord is a zombie it has already stopped
+            # accepting work, so the caller can clear stale pid/socket files
+            # without blocking for a process it cannot reap itself.
+            kill_remaining_processes(children, timeout=kill_timeout)
+            return
+        proc.wait(timeout=timeout)
+    except psutil.TimeoutExpired:
+        proc.kill()
+    kill_remaining_processes(children, timeout=kill_timeout)
+    try:
+        proc.wait(timeout=kill_timeout)
+    except (psutil.NoSuchProcess, psutil.TimeoutExpired):
+        pass
+
+
+def kill_remaining_processes(processes: list[psutil.Process], *, timeout: float = 2.0) -> None:
+    _gone, alive = psutil.wait_procs(processes, timeout=timeout)
+    for process in alive:
+        try:
+            process.kill()
+        except psutil.NoSuchProcess:
+            pass
+    psutil.wait_procs(alive, timeout=timeout)
+
+
+@contextmanager
+def foreground_shutdown_signals(
+    handled_signals: tuple[signal.Signals, ...] = (signal.SIGHUP, signal.SIGINT, signal.SIGTERM),
+    *,
+    first_signal_message: str | None = "\n[🛑] Got {signal_name}, stopping gracefully...\n",
+    on_signal: Callable[[signal.Signals], None] | None = None,
+    raise_on_first_signal: bool = True,
+) -> Iterator[ShutdownSignalState]:
+    """Install foreground signal handlers that print an immediate exit notice.
+
+    Some log-tail loops intentionally swallow KeyboardInterrupt so that callers
+    can centralize cleanup in finally blocks. The handler writes the signal name
+    immediately, then raises KeyboardInterrupt to break out of the blocking read.
+    """
+
+    global _active_shutdown_state
+
+    if threading.current_thread() is not threading.main_thread():
+        yield ShutdownSignalState()
+        return
+
+    state = ShutdownSignalState()
+    previous_active_state = _active_shutdown_state
+    previous_handlers = {sig: signal.getsignal(sig) for sig in handled_signals}
+
+    def raise_keyboard_interrupt(signum, _frame):
+        sig = signal.Signals(signum)
+        already_requested = state.signal_name is not None
+        if not already_requested:
+            state.signal_name = sig.name
+            if first_signal_message is not None:
+                os.write(sys.stdout.fileno(), first_signal_message.format(signal_name=state.signal_name).encode())
+        if on_signal is not None:
+            on_signal(sig)
+        # Foreground `archivebox add` uses the first signal to abort the active
+        # hook through the bus-facing runner code, then reserves the second
+        # signal for hard foreground-command shutdown. Server/update/run and
+        # other non-interactive commands raise immediately so their finally
+        # blocks can stop owned children without prompting.
+        if raise_on_first_signal or already_requested:
+            raise KeyboardInterrupt
+
+    try:
+        _active_shutdown_state = state
+        for sig in handled_signals:
+            signal.signal(sig, raise_keyboard_interrupt)
+        yield state
+    finally:
+        _active_shutdown_state = previous_active_state
+        for sig, previous_handler in previous_handlers.items():
+            signal.signal(sig, previous_handler)
+
+
+@contextmanager
+def foreground_parent_watchdog(
+    *,
+    enabled: bool = True,
+    check_interval: float = 2.0,
+    shutdown_signal: signal.Signals = signal.SIGTERM,
+) -> Iterator[None]:
+    """Ask a foreground command to exit if its launcher/wrapper disappears.
+
+    `uv run archivebox ...` and similar wrappers can be killed without
+    delivering a signal to the real Python child. If that child keeps crawling
+    as an orphan, it can hold SQLite write locks long after the user-facing
+    command timed out. This watchdog is only for foreground command lifetimes;
+    daemon/supervisord workers should not use it because their parent may
+    intentionally hand them off.
+    """
+
+    original_ppid = os.getppid()
+    if not enabled or original_ppid <= 1:
+        yield
+        return
+
+    stopped = threading.Event()
+
+    def watch_parent() -> None:
+        while not stopped.wait(check_interval):
+            if os.getppid() == original_ppid:
+                continue
+            sys.stderr.write("\n[🛑] ArchiveBox parent process exited; stopping foreground command gracefully...\n")
+            sys.stderr.flush()
+            os.kill(os.getpid(), shutdown_signal)
+            return
+
+    thread = threading.Thread(target=watch_parent, name="archivebox-parent-watchdog", daemon=True)
+    thread.start()
+    try:
+        yield
+    finally:
+        stopped.set()

archivebox/core/sqlite_backend/base.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import sqlite3
+import time
+from collections.abc import Mapping
+from itertools import tee
+import re
+
+from django.db.backends.sqlite3.base import DatabaseWrapper as DjangoSQLiteDatabaseWrapper
+from django.db.backends.sqlite3.base import SQLiteCursorWrapper as DjangoSQLiteCursorWrapper
+
+
+def _sqlite_lock_retry_timeout() -> float:
+    from django.conf import settings
+
+    return settings.CONFIG.SQLITE_LOCK_RETRY_TIMEOUT
+
+
+def _sqlite_lock_retry_interval() -> float:
+    from django.conf import settings
+
+    return settings.CONFIG.SQLITE_LOCK_RETRY_INTERVAL
+
+
+def _format_sql(query: str, params=None) -> str:
+    compact = " ".join(str(query).split())
+    match = re.match(r'^(INSERT INTO|UPDATE|DELETE FROM|SELECT) "?([A-Za-z0-9_]+)"?', compact, flags=re.IGNORECASE)
+    if match:
+        compact = f"{match.group(1).upper()} {match.group(2)}"
+    if params is not None:
+        if isinstance(params, str):
+            params_summary = params
+        elif isinstance(params, (tuple, list)):
+            preview = ", ".join(repr(param)[:60] for param in params[:4])
+            params_summary = f"{len(params)} params: {preview}"
+        elif isinstance(params, Mapping):
+            preview = ", ".join(f"{key}={repr(value)[:60]}" for key, value in list(params.items())[:4])
+            params_summary = f"{len(params)} params: {preview}"
+        else:
+            params_summary = repr(params)[:120]
+        compact = f"{compact} ({params_summary})"
+    return compact[:260]
+
+
+def _log_locked_database(query: str, params=None, *, attempt: int, elapsed: float, retry_interval: float) -> None:
+    from rich.console import Console
+
+    from archivebox.misc.db import log_sqlite_lock_holders
+
+    console = Console(stderr=True)
+    console.print(
+        f"[yellow][*] SQLite database is locked for {elapsed:.0f}s; retrying in {retry_interval:g}s... attempt={attempt}[/yellow]",
+    )
+    console.print(f"[yellow]    Query: {_format_sql(query, params)}[/yellow]")
+    log_sqlite_lock_holders(console)
+
+
+def _connection_in_transaction(connection) -> bool:
+    try:
+        return bool(connection and connection.in_transaction)
+    except (AttributeError, sqlite3.Error):
+        return False
+
+
+def _recover_non_atomic_connection(db_wrapper, query: str) -> None:
+    if db_wrapper is None or getattr(db_wrapper, "in_atomic_block", False):
+        return
+    connection = getattr(db_wrapper, "connection", None)
+    if not _connection_in_transaction(connection):
+        return
+    try:
+        connection.rollback()
+    except sqlite3.Error:
+        return
+
+
+def _is_inside_atomic(db_wrapper) -> bool:
+    return bool(db_wrapper is not None and getattr(db_wrapper, "in_atomic_block", False))
+
+
+def _abort_locked_database(query: str, params=None, *, elapsed: float, db_wrapper=None) -> None:
+    _recover_non_atomic_connection(db_wrapper, query)
+    raise sqlite3.OperationalError(
+        f"SQLite database remained locked for {elapsed:.0f}s while running {_format_sql(query, params)}; "
+        "aborting instead of retrying indefinitely",
+    )
+
+
+def _retry_locked_database(action, query: str, params=None, *, db_wrapper=None):
+    attempt = 0
+    started_at = time.monotonic()
+    while True:
+        try:
+            return action()
+        except (sqlite3.OperationalError, Exception) as err:
+            from archivebox.misc.db import sqlite_lock_error
+
+            if not sqlite_lock_error(err):
+                raise
+            attempt += 1
+            elapsed = time.monotonic() - started_at
+            retry_timeout = _sqlite_lock_retry_timeout()
+            retry_interval = _sqlite_lock_retry_interval()
+            _log_locked_database(query, params, attempt=attempt, elapsed=elapsed, retry_interval=retry_interval)
+            # If SQLite raised while Django is in autocommit mode, do not keep a
+            # partially-open sqlite transaction around while waiting. Explicit
+            # transaction.atomic() callers keep their normal transaction boundary.
+            _recover_non_atomic_connection(db_wrapper, query)
+            if _is_inside_atomic(db_wrapper):
+                raise
+            if retry_timeout and elapsed >= retry_timeout:
+                _abort_locked_database(query, params, elapsed=elapsed, db_wrapper=db_wrapper)
+            time.sleep(retry_interval)
+
+
+class SQLiteCursorWrapper(DjangoSQLiteCursorWrapper):
+    def execute(self, query, params=None):
+        if params is None:
+            return _retry_locked_database(
+                lambda: super(SQLiteCursorWrapper, self).execute(query),
+                query,
+                db_wrapper=getattr(self, "db_wrapper", None),
+            )
+        param_names = list(params) if isinstance(params, Mapping) else None
+        converted_query = self.convert_query(query, param_names=param_names)
+        return _retry_locked_database(
+            lambda: super(DjangoSQLiteCursorWrapper, self).execute(converted_query, params),
+            converted_query,
+            params,
+            db_wrapper=getattr(self, "db_wrapper", None),
+        )
+
+    def executemany(self, query, param_list):
+        peekable, param_list = tee(iter(param_list))
+        if (params := next(peekable, None)) and isinstance(params, Mapping):
+            param_names = list(params)
+        else:
+            param_names = None
+        converted_query = self.convert_query(query, param_names=param_names)
+        param_list = tuple(param_list)
+        return _retry_locked_database(
+            lambda: super(DjangoSQLiteCursorWrapper, self).executemany(converted_query, param_list),
+            converted_query,
+            f"{len(param_list)} parameter sets",
+            db_wrapper=getattr(self, "db_wrapper", None),
+        )
+
+
+class DatabaseWrapper(DjangoSQLiteDatabaseWrapper):
+    def create_cursor(self, name=None):
+        cursor = self.connection.cursor(factory=SQLiteCursorWrapper)
+        cursor.db_wrapper = self
+        return cursor
+
+    def _commit(self):
+        return _retry_locked_database(lambda: super(DatabaseWrapper, self)._commit(), "COMMIT", db_wrapper=self)
+
+    def _rollback(self):
+        return _retry_locked_database(lambda: super(DatabaseWrapper, self)._rollback(), "ROLLBACK", db_wrapper=self)

archivebox/core/tag_util.py

@@ -6,13 +6,13 @@ from typing import Any
 from urllib.parse import unquote
 
 from django.contrib.auth.models import User
-from django.db.models import Count, F, QuerySet
+from django.db.models import Count, Exists, F, OuterRef, QuerySet
 from django.db.models.functions import Lower
 from django.http import HttpRequest
 from django.urls import reverse
 
 from archivebox.config.common import get_config
-from archivebox.core.host_utils import build_snapshot_url, build_web_url
+from archivebox.core.routes_util import build_snapshot_url, build_web_url
 from archivebox.core.models import Snapshot, SnapshotTag, Tag
 
 
@@ -61,11 +61,10 @@ def get_matching_tags(
     created_by: str = "",
     year: str = "",
     has_snapshots: str = "all",
-    with_snapshot_counts: bool = True,
 ) -> QuerySet[Tag]:
     sort = normalize_tag_sort(sort)
     has_snapshots = normalize_has_snapshots_filter(has_snapshots)
-    needs_snapshot_counts = with_snapshot_counts or sort.startswith("snapshots_") or has_snapshots != "all"
+    needs_snapshot_counts = sort.startswith("snapshots_")
 
     queryset = Tag.objects.select_related("created_by")
     if needs_snapshot_counts:
@@ -83,10 +82,13 @@ def get_matching_tags(
     if year:
         queryset = queryset.filter(created_at__year=int(year))
 
+    if has_snapshots != "all" and not needs_snapshot_counts:
+        queryset = queryset.annotate(has_snapshot=Exists(SnapshotTag.objects.filter(tag_id=OuterRef("pk"))))
+
     if has_snapshots == "yes":
-        queryset = queryset.filter(num_snapshots__gt=0)
+        queryset = queryset.filter(num_snapshots__gt=0) if needs_snapshot_counts else queryset.filter(has_snapshot=True)
     elif has_snapshots == "no":
-        queryset = queryset.filter(num_snapshots=0)
+        queryset = queryset.filter(num_snapshots=0) if needs_snapshot_counts else queryset.filter(has_snapshot=False)
 
     if sort == "name_asc":
         queryset = queryset.order_by(Lower("name"), "id")
@@ -104,15 +106,15 @@ def get_matching_tags(
     return queryset
 
 
-def add_snapshot_counts(tags: list[Tag]) -> None:
+def add_snapshot_counts(tags: list[Tag], snapshot_queryset: QuerySet[Snapshot] | None = None) -> None:
     tag_ids = [tag.pk for tag in tags]
     if not tag_ids:
         return
 
-    counts = {
-        row["tag_id"]: row["num_snapshots"]
-        for row in SnapshotTag.objects.filter(tag_id__in=tag_ids).values("tag_id").annotate(num_snapshots=Count("snapshot_id"))
-    }
+    queryset = SnapshotTag.objects.filter(tag_id__in=tag_ids)
+    if snapshot_queryset is not None:
+        queryset = queryset.filter(snapshot_id__in=snapshot_queryset.values("id"))
+    counts = {row["tag_id"]: row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))}
     for tag in tags:
         tag.num_snapshots = counts.get(tag.pk, 0)
 
@@ -252,12 +254,12 @@ def build_tag_card(tag: Tag, snapshot_previews: list[dict[str, Any]] | None = No
         "name": tag.name,
         "slug": tag.slug,
         "num_snapshots": count,
-        "filter_url": f"{reverse('admin:core_snapshot_changelist')}?tags__id__exact={tag.pk}",
-        "edit_url": reverse("admin:core_tag_change", args=[tag.pk]),
-        "export_urls_url": reverse("api-1:tag_urls_export", args=[tag.pk]),
-        "export_jsonl_url": reverse("api-1:tag_snapshots_export", args=[tag.pk]),
-        "rename_url": reverse("api-1:rename_tag", args=[tag.pk]),
-        "delete_url": reverse("api-1:delete_tag", args=[tag.pk]),
+        "filter_url": f"/admin/core/snapshot/?tags__id__exact={tag.pk}",
+        "edit_url": f"/admin/core/tag/{tag.pk}/change/",
+        "export_urls_url": f"/api/v1/core/tag/{tag.pk}/urls.txt",
+        "export_jsonl_url": f"/api/v1/core/tag/{tag.pk}/snapshots.jsonl",
+        "rename_url": f"/api/v1/core/tag/{tag.pk}/rename",
+        "delete_url": f"/api/v1/core/tag/{tag.pk}/",
         "snapshots": snapshot_previews or [],
     }
 
@@ -274,14 +276,13 @@ def build_tag_cards(
 ) -> list[dict[str, Any]]:
     sort = normalize_tag_sort(sort)
     has_snapshots = normalize_has_snapshots_filter(has_snapshots)
-    needs_snapshot_count_annotation = sort.startswith("snapshots_") or has_snapshots != "all"
+    needs_snapshot_count_annotation = sort.startswith("snapshots_")
     queryset = get_matching_tags(
         query=query,
         sort=sort,
         created_by=created_by,
         year=year,
         has_snapshots=has_snapshots,
-        with_snapshot_counts=needs_snapshot_count_annotation,
     )
     if limit is not None:
         queryset = queryset[:limit]

archivebox/core/takeover_util.py

@@ -0,0 +1,294 @@
+from __future__ import annotations
+
+import time
+import sys
+from pathlib import Path
+
+from django.db import IntegrityError
+from django.utils import timezone
+from archivebox.config import CONSTANTS
+from archivebox.config.common import rprint
+
+RUNNER_ACTIVE_WORKER_TYPE = "worker_runner"
+RUNNER_WAITING_WORKER_TYPE = "runner_waiting"
+RUNNER_GATE_WORKER_TYPES = (RUNNER_ACTIVE_WORKER_TYPE, RUNNER_WAITING_WORKER_TYPE, "")
+
+
+def runtime_stack_owner_types():
+    from archivebox.machine.models import Process
+
+    return (
+        Process.TypeChoices.UPDATE,
+        Process.TypeChoices.SERVER,
+        Process.TypeChoices.ORCHESTRATOR,
+        Process.TypeChoices.ADD,
+    )
+
+
+def current_command(process_type: str, *, data_dir: str | Path, url: str | None = None):
+    from archivebox.machine.models import Process
+
+    proc = Process.current()
+    proc.mark_running(process_type=process_type, pwd=str(data_dir), url=url, timeout=CONSTANTS.MAX_HOOK_RUNTIME_SECONDS)
+    return proc
+
+
+def live_processes(*, process_type: str, data_dir: str | Path, url: str | None = None):
+    from archivebox.machine.models import Machine, Process
+
+    qs = Process.objects.filter(
+        machine=Machine.current(),
+        process_type=process_type,
+        status=Process.StatusChoices.RUNNING,
+        pwd=str(data_dir),
+    )
+    if url is not None:
+        qs = qs.filter(url=url)
+    return [proc for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50) if proc.is_running]
+
+
+def newest_live_process(*, process_type: str, data_dir: str | Path, url: str | None = None):
+    processes = live_processes(process_type=process_type, data_dir=data_dir, url=url)
+    return processes[0] if processes else None
+
+
+def command_is_newest(command, *, process_type: str, data_dir: str | Path, url: str | None = None) -> bool:
+    leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
+    return bool(leader and leader.id == command.id)
+
+
+def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
+    from archivebox.machine.models import Machine, Process
+
+    machine = Machine.current()
+    base_qs = Process.objects.filter(
+        machine=machine,
+        status=Process.StatusChoices.RUNNING,
+        pwd=str(data_dir),
+        process_type__in=runtime_stack_owner_types(),
+    )
+    if exclude_id is not None:
+        base_qs = base_qs.exclude(id=exclude_id)
+
+    top_level_types = (
+        Process.TypeChoices.UPDATE,
+        Process.TypeChoices.SERVER,
+        Process.TypeChoices.ADD,
+    )
+    for qs in (
+        base_qs.filter(process_type__in=top_level_types),
+        # A foreground `archivebox run` process is allowed to own the runtime
+        # stack when no server/update/add parent is alive. A runner launched by
+        # supervisord is only a child worker; after its parent is killed it must
+        # not keep stealing leadership from the next foreground command.
+        base_qs.filter(process_type=Process.TypeChoices.ORCHESTRATOR).exclude(parent__process_type=Process.TypeChoices.SUPERVISORD),
+    ):
+        for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+            if proc.is_running:
+                return proc
+            proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+    return None
+
+
+def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:
+    owner = runtime_stack_owner(data_dir=data_dir)
+    return bool(owner and owner.id == command.id)
+
+
+def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
+    try:
+        from archivebox.config.common import get_config
+        from archivebox.workers.supervisord_util import active_supervisord_runtime_components
+
+        components = active_supervisord_runtime_components(config=get_config())
+    except Exception:
+        components = []
+
+    names = list(components)
+    if not names and owner is not None:
+        from archivebox.machine.models import Process
+
+        if owner.process_type in {Process.TypeChoices.SERVER, Process.TypeChoices.ADD}:
+            names = ["orchestrator", "server"]
+        elif owner.process_type in {Process.TypeChoices.UPDATE, Process.TypeChoices.ORCHESTRATOR}:
+            names = ["orchestrator"]
+
+    return ", ".join(dict.fromkeys(names)) or "runtime stack"
+
+
+def ensure_daemon_stack(*, reason: str = ""):
+    from archivebox.config.common import get_config
+    from archivebox.workers.supervisord_util import (
+        get_existing_supervisord_process,
+        get_or_create_supervisord_process,
+        get_sonic_supervisord_worker_from_plugin,
+        get_worker,
+        start_worker,
+    )
+
+    config = get_config()
+    sonic_worker = get_sonic_supervisord_worker_from_plugin(config)
+    if sonic_worker is None:
+        return None
+
+    from abx_plugins.plugins.search_backend_sonic.daemon import is_port_listening, prepare_sonic_daemon
+
+    sonic_event = prepare_sonic_daemon(config)
+    if is_port_listening(sonic_event.host, sonic_event.port):
+        return {
+            "name": sonic_event.worker_name,
+            "statename": "RUNNING",
+            "description": f"existing Sonic daemon at {sonic_event.url}",
+        }
+
+    supervisor = get_existing_supervisord_process() or get_or_create_supervisord_process(daemonize=False)
+    worker = get_worker(supervisor, sonic_worker["name"])
+    if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"):
+        return worker
+
+    if reason:
+        rprint(f"[yellow][*] Starting daemon stack for {reason}...[/yellow]")
+    return start_worker(supervisor, sonic_worker)
+
+
+def healthy_orchestrator(*, data_dir: str | Path):
+    from archivebox.machine.models import Machine, Process
+    from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
+
+    supervisor = get_existing_supervisord_process()
+    worker = get_worker(supervisor, "worker_runner") if supervisor else None
+    if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"):
+        return worker
+
+    for proc in Process.objects.filter(
+        machine=Machine.current(),
+        process_type=Process.TypeChoices.ORCHESTRATOR,
+        status=Process.StatusChoices.RUNNING,
+        pwd=str(data_dir),
+    ).order_by("-created_at"):
+        if proc.is_running:
+            return proc
+    return None
+
+
+def _runner_sort_key(process):
+    return (process.started_at or process.created_at, process.created_at, str(process.id))
+
+
+def live_runner_processes(*, data_dir: str | Path, exclude_id=None):
+    from archivebox.machine.models import Machine, Process
+
+    machine = Machine.current()
+    Process.cleanup_stale_running(machine=machine)
+    qs = Process.objects.filter(
+        machine=machine,
+        status=Process.StatusChoices.RUNNING,
+        process_type=Process.TypeChoices.ORCHESTRATOR,
+        worker_type__in=RUNNER_GATE_WORKER_TYPES,
+        pwd=str(data_dir),
+    )
+    if exclude_id is not None:
+        qs = qs.exclude(id=exclude_id)
+    return [process for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20) if process.is_running]
+
+
+def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout: float = 5.0) -> bool:
+    """
+    Admit exactly one active runner for this DATA_DIR using Process rows.
+
+    The current process is a real OS process while it waits, so we keep its
+    Process row RUNNING but mark worker_type=runner_waiting. Only the process
+    that wins takeover is promoted to worker_type=worker_runner, which is
+    protected by a partial unique DB constraint. Older runners are terminated
+    and fully waited out before promotion, so the runner work loop never overlaps.
+    """
+    from archivebox.machine.models import Process
+
+    command.mark_running(
+        process_type=Process.TypeChoices.ORCHESTRATOR,
+        worker_type=RUNNER_WAITING_WORKER_TYPE,
+        pwd=str(data_dir),
+        timeout=CONSTANTS.MAX_HOOK_RUNTIME_SECONDS,
+    )
+    while True:
+        runners = live_runner_processes(data_dir=data_dir)
+        if all(process.id != command.id for process in runners):
+            command.refresh_from_db()
+            command.mark_running(
+                process_type=Process.TypeChoices.ORCHESTRATOR,
+                worker_type=RUNNER_WAITING_WORKER_TYPE,
+                pwd=str(data_dir),
+                timeout=CONSTANTS.MAX_HOOK_RUNTIME_SECONDS,
+            )
+            runners = live_runner_processes(data_dir=data_dir)
+
+        newest = max(runners, key=_runner_sort_key)
+        if newest.id != command.id:
+            rprint(
+                f"[yellow][*] Newer ArchiveBox runner pid={newest.pid} is taking over; exiting this runner.[/yellow]",
+                file=sys.stderr,
+            )
+            return False
+
+        older_runners = [process for process in runners if process.id != command.id]
+        if older_runners:
+            for process in older_runners:
+                rprint(f"[yellow][*] Stopping older ArchiveBox runner process (pid={process.pid})...[/yellow]", file=sys.stderr)
+                process.kill_tree(graceful_timeout=graceful_timeout)
+            time.sleep(0.1)
+            continue
+
+        try:
+            command.mark_running(
+                process_type=Process.TypeChoices.ORCHESTRATOR,
+                worker_type=RUNNER_ACTIVE_WORKER_TYPE,
+                pwd=str(data_dir),
+                timeout=CONSTANTS.MAX_HOOK_RUNTIME_SECONDS,
+            )
+            return True
+        except IntegrityError:
+            # A hard-killed runner may leave the unique active-runner row behind.
+            # The next loop starts by proving each RUNNING Process row still maps
+            # to a live OS process, then marks dead rows EXITED before retrying.
+            command.refresh_from_db()
+            time.sleep(0.1)
+
+
+def standby_until_leader_needed(command, *, process_type: str, data_dir: str | Path, url: str | None = None, interval: float = 2.0) -> None:
+    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
+
+    announced = False
+    while not command_is_newest(command, process_type=process_type, data_dir=data_dir, url=url):
+        reap_foreground_supervisord_process()
+        if not announced:
+            leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
+            leader_pid = leader.pid if leader else "unknown"
+            rprint(f"[yellow][*] Standing by; newer ArchiveBox process pid={leader_pid} is running the orchestrator and server.[/yellow]")
+            announced = True
+        time.sleep(interval)
+    command.modified_at = timezone.now()
+    command.save(update_fields=["modified_at"])
+
+
+def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> dict[str, object]:
+    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
+
+    announced = False
+    previous_owner_pid = None
+    while not command_owns_runtime_stack(command, data_dir=data_dir):
+        reap_foreground_supervisord_process()
+        if not announced:
+            owner = runtime_stack_owner(data_dir=data_dir)
+            owner_pid = owner.pid if owner else "unknown"
+            components = runtime_stack_component_label(owner=owner, data_dir=data_dir)
+            previous_owner_pid = owner_pid
+            rprint(
+                f"[yellow][*] A newer archivebox process took over the {components} "
+                f"(pid={owner_pid}). Work will continue there, and will resume here if that process exits and work still remains.[/yellow]",
+                file=sys.stderr,
+            )
+            announced = True
+        time.sleep(interval)
+    command.modified_at = timezone.now()
+    command.save(update_fields=["modified_at"])
+    return {"resumed": announced, "previous_owner_pid": previous_owner_pid}

archivebox/core/templatetags/config_tags.py

@@ -1,22 +0,0 @@
-"""Template tags for accessing config values in templates."""
-
-from typing import Any
-
-from django import template
-
-from archivebox.config.common import get_config
-
-register = template.Library()
-
-
-@register.simple_tag(name="get_config")
-def get_config_tag(key: str) -> Any:
-    """
-    Get a config value by key.
-
-    Usage: {% get_config "ARCHIVEDOTORG_ENABLED" as enabled %}
-    """
-    try:
-        return get_config().get(key)
-    except (KeyError, AttributeError):
-        return None

archivebox/core/templatetags/core_tags.py

@@ -7,12 +7,15 @@ from django.utils.html import escape
 
 from pathlib import Path
 
-from archivebox.hooks import (
+from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target
+
+from archivebox.plugins.discovery import (
     get_plugin_icon,
     get_plugin_template,
     get_plugin_name,
 )
-from archivebox.core.host_utils import (
+from archivebox.core.routes_util import (
+    canonical_base_host_for_request,
     get_admin_base_url,
     get_public_base_url,
     get_web_base_url,
@@ -204,7 +207,12 @@ def _build_snapshot_preview_url(snapshot_id: str, path: str = "", request=None,
     if _is_root_snapshot_output_path(path):
         return _build_snapshot_files_url(snapshot_id, request=request, config=config)
     url = build_snapshot_url(str(snapshot_id), path, request=request, config=config)
-    if not (_is_text_preview_path(path) or _is_image_preview_path(path) or (path or "").lower().endswith(_MHTML_PREVIEW_EXTS)):
+    if not (
+        _is_text_preview_path(path)
+        or _is_image_preview_path(path)
+        or (path or "").lower().endswith(_MHTML_PREVIEW_EXTS)
+        or is_archivewebpage_replay_target(path or "")
+    ):
         return url
     separator = "&" if "?" in url else "?"
     return f"{url}{separator}preview=1"
@@ -278,6 +286,14 @@ def file_size(num_bytes: int | float) -> str:
     return "{:3.1f} {}".format(num_bytes, "TB")
 
 
+@register.filter
+def intcomma(value: int | str | None) -> str:
+    try:
+        return f"{int(value or 0):,}"
+    except (TypeError, ValueError):
+        return str(value or "")
+
+
 def result_list(context, cl):
     """
     Monkey patched result
@@ -309,6 +325,143 @@ def result_list_tag(parser, token):
     )
 
 
+_LOW_DISK_THRESHOLD_GB = 1.0
+_HIGH_MEMORY_THRESHOLD_PCT = 95.0
+_HIGH_LOAD_MULTIPLE = 3  # 15-min loadavg > 3 * cpu_count
+_HEALTH_CHECK_INTERVAL_SECONDS = 30
+_health_cache: dict = {"checked_at": 0.0, "stats": {}}
+
+
+def _machine_health_stats() -> dict:
+    """Cached wrapper around the Machine-admin stats util.
+
+    The Machine list/change pages already render disk / mem / load via
+    ``archivebox.machine.detect.get_host_stats()`` — we reuse the same call so
+    the banner thresholds line up 1:1 with what's shown on /admin/machine/.
+    Cached for 30s because the inclusion tag fires on every page render and
+    ``get_host_stats`` shells into several psutil probes.
+    """
+    import time
+
+    now = time.monotonic()
+    if _health_cache["stats"] and (now - _health_cache["checked_at"]) < _HEALTH_CHECK_INTERVAL_SECONDS:
+        return _health_cache["stats"]
+
+    try:
+        from archivebox.machine.detect import get_host_stats
+
+        stats = get_host_stats() or {}
+    except Exception:
+        stats = {}
+
+    _health_cache["checked_at"] = now
+    _health_cache["stats"] = stats
+    return stats
+
+
+@register.inclusion_tag("system_warnings_banner.html", takes_context=True)
+def system_warnings_banner(context):
+    """Render the top-of-page warning banner for one of the conditions below,
+    in priority order (highest first):
+
+    1. ``mode="unconfigured"``— ``BASE_URL`` is empty. Security/correctness
+       issue: until it's pinned, generated URLs can echo any Host the client
+       sends, and admin/web/api routing has no canonical anchor.
+    2. ``mode="unsafe"``      — ``SERVER_SECURITY_MODE`` is a non-subdomain
+       mode. Archived pages share an origin with privileged routes.
+    3. ``mode="low_disk"``    — ``DATA_DIR`` has <1 GiB free; new archive
+       jobs will start failing on ENOSPC.
+    4. ``mode="high_memory"`` — virtual memory utilization at/above 95%; the
+       host is one OOM-kill from a crash.
+    5. ``mode="high_load"``   — 15-minute load average exceeds 3 × CPU count
+       (the kernel's own sustained-load EMA, so no rolling buffer of ours is
+       needed).
+
+    Config/security warnings come first because they affect correctness +
+    security and need explicit operator action; host-health warnings come
+    after and reuse ``machine.detect.get_host_stats`` (the same function that
+    populates the Machine admin page), cached for 30s.
+    """
+    config = context.get("CONFIG")
+    if config is None:
+        from archivebox.config.common import get_config
+
+        config = get_config(resolve_plugins=False)
+
+    if not config.BASE_URL:
+        return _unconfigured_banner_context(context.get("request"))
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return {"mode": "unsafe"}
+
+    stats = _machine_health_stats()
+    free_gb = stats.get("disk_data_free_gb")
+    if isinstance(free_gb, (int, float)) and free_gb < _LOW_DISK_THRESHOLD_GB:
+        return {"mode": "low_disk", "free_gb": f"{free_gb:.2f}"}
+
+    mem_pct = stats.get("mem_virt_used_pct")
+    if isinstance(mem_pct, (int, float)) and mem_pct >= _HIGH_MEMORY_THRESHOLD_PCT:
+        return {"mode": "high_memory", "mem_pct": f"{mem_pct:.1f}"}
+
+    cpu_load = stats.get("cpu_load") or ()
+    cpu_count = stats.get("cpu_count") or 1
+    # ``cpu_load`` is the (1min, 5min, 15min) tuple from psutil.getloadavg();
+    # we take the 15-min figure because the operator's threshold was
+    # "sustained for 15min" and the kernel already maintains that EMA.
+    load_15 = cpu_load[2] if isinstance(cpu_load, (list, tuple)) and len(cpu_load) >= 3 else None
+    if isinstance(load_15, (int, float)) and load_15 > _HIGH_LOAD_MULTIPLE * cpu_count:
+        return {
+            "mode": "high_load",
+            "load_15": f"{load_15:.2f}",
+            "cpu_count": cpu_count,
+            "load_threshold": _HIGH_LOAD_MULTIPLE * cpu_count,
+        }
+
+    return {"mode": ""}
+
+
+def _unconfigured_banner_context(request) -> dict:
+    """Build the banner payload for the unset-BASE_URL case.
+
+    Always returns ``mode="unconfigured"`` — the user explicitly asked for
+    the banner to render whenever ``BASE_URL`` is empty, regardless of
+    whether the request host happens to match a CSRF-derived value. The
+    ``suggested_base_url`` is derived from the current request when one is
+    available so the user can copy/paste the right value straight into
+    their config.
+    """
+    if request is None:
+        return {
+            "mode": "unconfigured",
+            "actual_host": "",
+            "suggested_base_url": "",
+            "machine_admin_url": "",
+        }
+    scheme = request.scheme or "http"
+    actual_full_host = request.get_host() or ""
+    canonical_host = canonical_base_host_for_request(actual_full_host)
+    # Suggest the wildcard form ``http://*.<host>`` so the value lands in the
+    # operator's clipboard already aligned with subdomain routing. The config
+    # parser strips the leading ``*.`` so users can paste it verbatim.
+    suggested_base_url = f"{scheme}://*.{canonical_host}" if canonical_host else ""
+    user = getattr(request, "user", None)
+    is_superuser = bool(user and user.is_authenticated and user.is_superuser)
+    machine_admin_url = ""
+    if is_superuser:
+        try:
+            from archivebox.machine.models import Machine
+
+            machine = Machine.current()
+            machine_admin_url = f"/admin/machine/machine/{machine.id}/change/"
+        except Exception:
+            machine_admin_url = ""
+    return {
+        "mode": "unconfigured",
+        "actual_host": actual_full_host,
+        "suggested_base_url": suggested_base_url,
+        "machine_admin_url": machine_admin_url,
+    }
+
+
 @register.simple_tag(takes_context=True)
 def url_replace(context, **kwargs):
     dict_ = context["request"].GET.copy()
@@ -316,6 +469,16 @@ def url_replace(context, **kwargs):
     return dict_.urlencode()
 
 
+@register.simple_tag
+def has_real_admin_users() -> bool:
+    """True if any non-``system`` superuser exists. Used by the login page to
+    only show the bootstrap hint (createsuperuser / ADMIN_USERNAME env vars)
+    when the collection still has no real admin."""
+    from django.contrib.auth.models import User
+
+    return User.objects.filter(is_superuser=True).exclude(username="system").exists()
+
+
 @register.simple_tag(takes_context=True)
 def admin_base_url(context) -> str:
     return get_admin_base_url(request=context.get("request"), config=context.get("CONFIG"))

archivebox/core/urls.py

@@ -24,8 +24,8 @@ from archivebox.core.views import (
     AddView,
     WebAddView,
     HealthCheckView,
-    live_progress_view,
 )
+from archivebox.progressmonitor.views import live_progress_view
 
 
 # GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
@@ -66,10 +66,15 @@ urlpatterns = [
     ),
     path("admin/core/snapshot/add/", RedirectView.as_view(url="/add/")),
     path("add/", AddView.as_view(), name="add"),
-    path("accounts/login/", RedirectView.as_view(url="/admin/login/")),
-    path("accounts/logout/", RedirectView.as_view(url="/admin/logout/")),
+    # ``query_string=True`` preserves the ``?next=…`` param that Django's
+    # auth/login mixins append, so e.g. ``UserPassesTestMixin`` redirecting
+    # an unauthenticated ``/add`` visitor to ``/accounts/login/?next=/add/``
+    # carries the ``next`` through to ``/admin/login/`` and lands them at
+    # ``/add/`` after login instead of the admin homepage.
+    path("accounts/login/", RedirectView.as_view(url="/admin/login/", query_string=True)),
+    path("accounts/logout/", RedirectView.as_view(url="/admin/logout/", query_string=True)),
     path("accounts/", include("django.contrib.auth.urls")),
-    path("admin/live-progress/", archivebox_admin.admin_view(live_progress_view), name="live_progress"),
+    path("progress.json", live_progress_view, name="live_progress"),
     path("admin/", archivebox_admin.urls),
     path("api/", include("archivebox.api.urls"), name="api"),
     path("health/", HealthCheckView.as_view(), name="healthcheck"),