release: v0.9.35rc67

Clarify security risks and update information on CSRF/XSS protections.

.claude/settings.json

@@ -1,5 +0,0 @@
-{
-  "enabledPlugins": {
-    "pyright-lsp@claude-plugins-official": true
-  }
-}

.claude/settings.local.json

@@ -1,51 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Read(**)",
-      "Glob(**)",
-      "Grep(**)",
-      "Bash(python -m archivebox:*)",
-      "Bash(ls:*)",
-      "Bash(xargs:*)",
-      "Bash(python -c:*)",
-      "Bash(printf:*)",
-      "Bash(pkill:*)",
-      "Bash(python3:*)",
-      "Bash(sqlite3:*)",
-      "WebFetch(domain:github.com)",
-      "Bash(uv add:*)",
-      "Bash(mkdir:*)",
-      "Bash(chmod:*)",
-      "Bash(python -m forum_dl:*)",
-      "Bash(archivebox manage migrate:*)",
-      "Bash(cat:*)",
-      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
-      "Bash(forum-dl:*)",
-      "Bash(pip uninstall:*)",
-      "Bash(python:*)",
-      "Bash(source .venv/bin/activate)",
-      "Bash(mv:*)",
-      "Bash(echo:*)",
-      "Bash(grep:*)",
-      "WebFetch(domain:python-statemachine.readthedocs.io)",
-      "Bash(./bin/run_plugin_tests.sh:*)",
-      "Bash(done)",
-      "Bash(coverage erase:*)",
-      "Bash(gh api:*)"
-    ]
-  },
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Bash",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
-            "statusMessage": "Checking working directory..."
-          }
-        ]
-      }
-    ]
-  }
-}

.dockerignore

@@ -3,18 +3,33 @@
 /.heartbeat.json
 /package.json
 /package-lock.json
+__pycache__
+**/__pycache__
 *.pyc
-__pycache__/
+*.pyo
+*.py[cod]
+*$py.class
 .mypy_cache/
 .pytest_cache/
+.ruff_cache/
+.uv-cache/
+.cache/
+.cache*/
 .github/
 .pdm-build/
+.pdm-build/**
 .pdm-python
 .eggs/
+*.egg-info/
 .git/
+!.git/
+.git/*
 .vscode/
 !.git/HEAD
-!.git/refs/heads/*
+!.git/packed-refs
+!.git/refs/
+!.git/refs/heads/
+!.git/refs/heads/**
 
 venv/
 .venv/
@@ -31,6 +46,7 @@ chromeprofile/
 chrome_profile/
 lib/
 out/
+tests/out/
 users/
 archive/
 crawls/
@@ -38,6 +54,8 @@ snapshots/
 logs/
 archivebox-docker-smoke*/
 archivebox-compose-smoke*/
+docker-test/
+docker-test*/
 core
 *.core
 
@@ -50,6 +68,8 @@ dist/
 brew_dist/
 deb_dist/
 pip_dist/
+cache/
+caches/
 assets/
 docker/
 website/

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -69,7 +69,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:

.github/ISSUE_TEMPLATE/2-feature_request.yml

@@ -71,7 +71,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:

.github/SECURITY.md

@@ -2,18 +2,15 @@
 
 ---
 
-## Security Information
+## Security Documentation
 
-Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
-
-https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
-
-Also see this section of the README about important caveats when running ArchiveBox:
-
-https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
-
-You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
+Please start by reading these links to understand our existing security model, known issues, API surfaces, and setup steps:
 
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+- https://github.com/ArchiveBox/ArchiveBox/security/advisories/
+- https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication
 - https://github.com/ArchiveBox/ArchiveBox#archive-layout
 - https://github.com/ArchiveBox/ArchiveBox#archivebox-development
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
@@ -32,3 +29,43 @@ We use Github's built-in [Private Reporting](https://docs.github.com/en/code-sec
 3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers
 
 You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).
+
+---
+
+## CVEs and Report Credits
+
+> [!IMPORTANT]
+> **Reports that only affect `dev` or beta versions will be converted to normal Github issues and will not be issued CVEs.**  
+> Reporters **will still receive credit in release notes** however.
+
+> [!TIP]
+> Reports that affect **stable published versions will be issued CVEs** and published w/ full credit.
+
+Please read our existing published [security advisories](https://github.com/ArchiveBox/ArchiveBox/security/advisories), [Security Overview Docs](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview), and [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and avoid creating duplicates.
+
+---
+
+## Threat Model
+
+### All admin users are assumed to be trusted
+
+ArchiveBox does not support granular user permissions, there are only two types of users available in the Admin UI & REST API:
+
+- logged in admin superusers (which are assumed to have `root` on the ArchiveBox host machine already)
+- non-logged in anonymous guest users
+
+Because admin / REST API / CLI / Python API users are already assumed to have `root` on the host machine, there are deliberately no protections against admins inputting malicious config values, attempts to attack other admin users, or destructive/malicious actions. Permissions for non-logged in users are managed via the [`PERMISSIONS`](https://github.com/archiveBox/archiveBox/wiki/configuration#permissions) + [`PUBLIC_INDEX` + `PUBLIC_ADD_VIEW`](https://github.com/archiveBox/archiveBox/wiki/configuration#public_index--public_add_view) + other config options.
+
+### Sanitization of Config Values, Filesystem Paths, DB Fields, etc.
+
+Non-logged in users can only choose between public Persona config presets on the `/add/` page, but they should never be allowed to directly set config options like `*_ARGS`, `*_BINARY` or DB fields like `url`, `title`, `tag.name`, `notes` without sanitization/filtering on save and/or use.
+
+Non-logged in users should be tightly limited to prevent attacks like SQL/shell injection, path traversal attacks, XSS, cookie leaks between admin/non-admin domains, etc. Any reports related to non-logged-in users will be prioritized as this is a surface we care a lot about hardening.
+
+### XSS, CSRF, CORS, CSP
+
+`< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
+
+[`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
+
+Note archivebox is designed to be able to archive any URL the server can reach, including internal intranet URLs. The only URLs that are blocked are URLs hosted by the archivebox server itself.

.github/scripts/clone_abx_repo.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+repo_name="$1"
+target_dir="${2:-$repo_name}"
+
+case "$repo_name" in
+    abxpkg)
+        version="1.11.230"
+        ;;
+    abx-plugins)
+        version="1.11.238"
+        ;;
+    *)
+version="$(
+python3 - "$repo_name" <<'PY'
+import re
+import sys
+from pathlib import Path
+
+repo_name = sys.argv[1]
+lock_text = Path("uv.lock").read_text()
+match = re.search(
+    rf'^\[\[package\]\]\s*\nname = "{re.escape(repo_name)}"\s*\nversion = "([^"]+)"',
+    lock_text,
+    re.MULTILINE,
+)
+if not match:
+    raise SystemExit(f"Could not find {repo_name} in uv.lock")
+print(match.group(1))
+PY
+)"
+        ;;
+esac
+
+echo "Cloning ArchiveBox/${repo_name}@v${version} into ${target_dir}"
+git clone --depth=1 --branch "v${version}" "https://github.com/ArchiveBox/${repo_name}.git" "$target_dir"

.github/workflows/codeql.yml

@@ -19,6 +19,10 @@ on:
   schedule:
     - cron: '33 17 * * 6'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   analyze:
     name: Analyze (${{ matrix.language }})

.github/workflows/debian.yml

@@ -1,210 +0,0 @@
-name: Build Debian package
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'pkg/debian/**'
-      - 'bin/build_deb.sh'
-      - 'bin/release_deb.sh'
-      - '.github/workflows/debian.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: write
-
-jobs:
-  build:
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        arch: [amd64, arm64]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Install nfpm
-        env:
-          NFPM_VERSION: "2.45.1"
-        run: |
-          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \
-            | sudo tar -xz -C /usr/local/bin nfpm
-          nfpm --version
-
-      - name: Build .deb package
-        run: |
-          export VERSION="${{ steps.version.outputs.version }}"
-          export ARCH="${{ matrix.arch }}"
-          ./bin/build_deb.sh
-
-      - name: Verify .deb package contents
-        run: |
-          DEB_FILE="$(ls dist/archivebox*.deb | head -1)"
-          echo "=== Package info ==="
-          dpkg-deb --info "$DEB_FILE"
-          echo ""
-          echo "=== Package contents ==="
-          dpkg-deb --contents "$DEB_FILE"
-          echo ""
-          echo "=== Control fields ==="
-          dpkg-deb --field "$DEB_FILE"
-
-      - name: Upload .deb artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb
-          path: dist/*.deb
-
-  test:
-    needs: build
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-            runner: ubuntu-24.04
-          - arch: arm64
-            runner: ubuntu-24.04-arm
-    runs-on: ${{ matrix.runner }}
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local wheel
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --wheel --out-dir /tmp/wheels/
-
-      - name: Download .deb artifact
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*-${{ matrix.arch }}.deb
-          merge-multiple: true
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget
-
-      - name: Pre-seed virtualenv with local wheel before dpkg install
-        run: |
-          # CI-only: pre-seed the venv with a local wheel so we test the
-          # unreleased code, not whatever is on PyPI. We explicitly use python3.13
-          # here (matching the system dep installed above) to ensure the venv is
-          # created with the correct Python version. On real installs, install.sh
-          # handles this by preferring python3.13 and failing if < 3.13.
-          # When postinstall.sh runs during dpkg -i, it finds the venv already
-          # populated and just upgrades deps.
-          sudo mkdir -p /opt/archivebox
-          sudo python3.13 -m venv /opt/archivebox/venv
-          sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools
-          sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl
-          echo "[√] Pre-seeded /opt/archivebox/venv with local wheel"
-
-      - name: Install .deb package
-        run: |
-          # dpkg install will run postinstall which creates the user,
-          # sets up systemd, and tries pip install (which finds it already installed)
-          sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y
-
-      - name: Verify archivebox is installed
-        run: |
-          which archivebox
-          archivebox version
-
-      - name: Verify wrapper script works
-        run: |
-          /usr/bin/archivebox version
-          /usr/bin/archivebox --help | head -5
-
-      - name: Test archivebox init as archivebox user
-        run: |
-          # The postinstall should have created the user
-          id archivebox
-          sudo mkdir -p /tmp/archivebox-test
-          sudo chown archivebox:archivebox /tmp/archivebox-test
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init'
-
-      - name: Test archivebox status
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status'
-
-      - name: Test archivebox add
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"'
-
-      - name: Verify systemd service file exists
-        run: |
-          test -f /usr/lib/systemd/system/archivebox.service
-          cat /usr/lib/systemd/system/archivebox.service
-
-  # Upload .deb to GitHub Release only after tests pass
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: [build, test]
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: write
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Determine release tag
-        id: tag
-        run: |
-          TAG="${{ github.event.release.tag_name }}"
-          if [ -z "$TAG" ]; then
-            TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')"
-            echo "[i] No release tag in event context, using version from pyproject.toml: $TAG"
-          fi
-          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
-
-      - name: Download all .deb artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*.deb
-          merge-multiple: true
-
-      - name: Upload .deb to GitHub Release
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          TAG="${{ steps.tag.outputs.tag }}"
-          # Verify the release exists before uploading
-          if ! gh release view "$TAG" >/dev/null 2>&1; then
-            echo "[!] No GitHub Release found for tag $TAG."
-            if [ -n "${{ github.event.release.tag_name }}" ]; then
-              echo "[X] This was triggered by a release event — the release should exist. Failing."
-              exit 1
-            fi
-            echo "[i] Skipping upload (workflow_dispatch without a release)."
-            echo "    Create a release first or trigger via the release event."
-            exit 0
-          fi
-          gh release upload "$TAG" *.deb --clobber

.github/workflows/docker.yml

@@ -5,17 +5,39 @@ on:
   workflow_call:
   push:
     branches:
-      - '**'
+      - "**"
     tags:
       - 'v*'
   # pull_request:
 
 env:
-  DOCKER_IMAGE: archivebox-ci
+  DOCKERHUB_IMAGE: archivebox/archivebox
+  GHCR_IMAGE: ghcr.io/archivebox/archivebox
+
+permissions:
+  contents: read
+  packages: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  buildx:
-    runs-on: ubuntu-latest
+  build:
+    name: build ${{ matrix.platform }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-24.04
+            cache_scope: docker-amd64
+            artifact_name: digest-linux-amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            cache_scope: docker-arm64
+            artifact_name: digest-linux-arm64
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -23,8 +45,46 @@ jobs:
         #   submodules: true
         #   fetch-depth: 1
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+      - name: Wait for released ArchiveBox deps on PyPI
+        run: |
+          python3 - <<'PY'
+          import json
+          import re
+          import sys
+          import time
+          import tomllib
+          import urllib.request
+
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  try:
+                      with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20):
+                          available = True
+                  except Exception:
+                      available = False
+                  if available:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
+          PY
 
       - name: Set up Docker Buildx
         id: buildx
@@ -32,7 +92,6 @@ jobs:
         with:
           version: latest
           install: true
-          platforms: linux/amd64,linux/arm64
 
       - name: Builder instance name
         run: echo ${{ steps.buildx.outputs.name }}
@@ -40,13 +99,40 @@ jobs:
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
-      - name: Cache Docker layers
-        uses: actions/cache@v4
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
+      - name: Wait for published abx-dl image
+        id: abx_dl_image
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          ABX_DL_VERSION="$(python3 - <<'PY'
+          import re
+          import tomllib
+
+          with open("pyproject.toml", "rb") as f:
+              deps = tomllib.load(f)["project"]["dependencies"]
+
+          for dep in deps:
+              match = re.match(r"abx-dl\s*(?:==|>=)\s*([^,;\s]+)", dep)
+              if match:
+                  print(match.group(1))
+                  break
+          else:
+              raise SystemExit("Missing abx-dl dependency in pyproject.toml")
+          PY
+          )"
+          ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
+          deadline=$((SECONDS + 1800))
+          until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
+            if (( SECONDS >= deadline )); then
+              echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2
+              exit 1
+            fi
+            echo "${ABX_DL_IMAGE} is not published yet; waiting..."
+            sleep 30
+          done
+
+          echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
+          docker buildx imagetools inspect "${ABX_DL_IMAGE}"
 
       - name: Login to Docker Hub
         uses: docker/login-action@v3
@@ -62,68 +148,239 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Collect Full Release Docker tags
-        # https://github.com/docker/metadata-action
+      - name: Collect Docker labels
         id: docker_meta
-        uses: docker/metadata-action@v5
-        if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
-        with:
-          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
-          tags: |
-              # :stable
-              type=ref,event=branch
-              # :0.7.3
-              type=semver,pattern={{version}}
-              # :0.7
-              type=semver,pattern={{major}}.{{minor}}
-              # :sha-463ea54
-              type=sha
-              # :latest
-              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$(python3 - <<'PY'
+          import tomllib
 
-      - name: Collect Non-Release Docker tags
-        # https://github.com/docker/metadata-action
-        id: docker_meta_non_release
-        uses: docker/metadata-action@v5
-        if: github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call'
-        with:
-          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
-          tags: |
-              # :stable
-              type=ref,event=branch
-              # :sha-463ea54
-              type=sha
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
+          PY
+          )"
 
-      - name: Build and push
+          {
+            echo 'labels<<EOF'
+            echo "org.opencontainers.image.version=${VERSION}"
+            echo "org.opencontainers.image.revision=${GITHUB_SHA}"
+            echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
+            echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
+            echo 'EOF'
+            echo "version=${VERSION}"
+          } >> "$GITHUB_OUTPUT"
+
+          echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
+
+      - name: Build and push digest
         id: docker_build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./
           file: ./Dockerfile
           builder: ${{ steps.buildx.outputs.name }}
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.tags || steps.docker_meta_non_release.outputs.tags }}
-          labels: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.labels || steps.docker_meta_non_release.outputs.labels }}
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new
-          platforms: linux/amd64,linux/arm64
+          tags: |
+            ${{ env.DOCKERHUB_IMAGE }}
+            ${{ env.GHCR_IMAGE }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+          build-args: |
+            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
+            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
+          cache-from: type=gha,scope=${{ matrix.cache_scope }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
+          pull: true
+          platforms: ${{ matrix.platform }}
+          outputs: type=image,push-by-digest=true,name-canonical=true,push=true
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
 
+      - name: Validate pushed image version, commit, and size
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          IMAGE="${DOCKERHUB_IMAGE}@${{ steps.docker_build.outputs.digest }}"
+          SHORT_SHA="${GITHUB_SHA::7}"
+
+          DATA_DIR="$(mktemp -d)"
+          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
+          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version | tee /tmp/archivebox-version.out
+          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
+          docker run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt > /tmp/archivebox-image-version.txt
+          grep -F "ArchiveBox v${{ steps.docker_meta.outputs.version }}" /tmp/archivebox-version.out
+          grep -F "COMMIT_HASH=${SHORT_SHA}" /tmp/archivebox-version.out
+          grep -Fx "COMMIT_HASH=${GITHUB_SHA}" /tmp/archivebox-image-version.txt
+
+          python3 - "$IMAGE" <<'PY'
+          import json
+          import subprocess
+          import sys
+
+          image = sys.argv[1]
+          limit = 780 * 1024 * 1024
+          manifest = json.loads(subprocess.check_output(["docker", "manifest", "inspect", image]))
+          total = manifest.get("config", {}).get("size", 0) + sum(layer.get("size", 0) for layer in manifest.get("layers", []))
+          print(f"{image} compressed_size={total / 1024 / 1024:.2f} MiB limit={limit / 1024 / 1024:.2f} MiB")
+          if total > limit:
+              raise SystemExit(f"{image} is over the compressed size limit")
+          PY
+
+      - name: Export digest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mkdir -p /tmp/digests
+          digest="${{ steps.docker_build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.artifact_name }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  publish:
+    name: publish multiarch tags
+    runs-on: ubuntu-24.04
+    needs:
+      - build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: latest
+          install: true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digest-*
+          merge-multiple: true
+
+      - name: Collect Docker tags
+        id: docker_meta
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$(python3 - <<'PY'
+          import tomllib
+
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
+          PY
+          )"
+          BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)"
+          SHORT_SHA="${GITHUB_SHA::12}"
+          test -n "$BRANCH_TAG"
+          test -n "$SHORT_SHA"
+
+          {
+            echo 'dockerhub_tags<<EOF'
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${DOCKERHUB_IMAGE}:latest"
+            fi
+            echo "${DOCKERHUB_IMAGE}:${BRANCH_TAG}"
+            echo "${DOCKERHUB_IMAGE}:${VERSION}"
+            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+            echo 'EOF'
+            echo 'ghcr_tags<<EOF'
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${GHCR_IMAGE}:latest"
+            fi
+            echo "${GHCR_IMAGE}:${BRANCH_TAG}"
+            echo "${GHCR_IMAGE}:${VERSION}"
+            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+            echo 'EOF'
+            echo "version=${VERSION}"
+            echo "branch_tag=${BRANCH_TAG}"
+            echo "short_sha=${SHORT_SHA}"
+          } >> "$GITHUB_OUTPUT"
+
+          echo "[+] Publishing Docker Hub tags:"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${DOCKERHUB_IMAGE}:latest"
+          fi
+          printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+          echo "[+] Publishing GHCR tags:"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${GHCR_IMAGE}:latest"
+          fi
+          printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+
+      - name: Create Docker Hub manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
+          [[ "${#DIGESTS[@]}" -gt 0 ]]
+
+          TAG_ARGS=()
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
+          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
+
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do
+            REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}")
+          done
+
+          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Create GHCR manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
+          [[ "${#DIGESTS[@]}" -gt 0 ]]
+
+          TAG_ARGS=()
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
+          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
+
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do
+            REFS+=("${GHCR_IMAGE}@sha256:${digest}")
+          done
+
+          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Inspect published images
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
+          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
+          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
+
       - name: Update README
         uses: peter-evans/dockerhub-description@v4
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: archivebox/archivebox
-
-      # This ugly bit is necessary if you don't want your cache to grow forever
-      # until it hits GitHub's limit of 5GB.
-      # Temp fix
-      # https://github.com/docker/build-push-action/issues/252
-      # https://github.com/moby/buildkit/issues/1896
-      - name: Move cache
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/homebrew.yml

@@ -1,229 +0,0 @@
-name: Build Homebrew formula
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'brew_dist/**'
-      - 'bin/build_brew.sh'
-      - 'bin/release_brew.sh'
-      - '.github/workflows/homebrew.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: read
-
-jobs:
-  build-and-test:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-latest, ubuntu-24.04]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Validate formula template syntax
-        run: ruby -c brew_dist/archivebox.rb
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies (Linux)
-        if: runner.os == 'Linux'
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local sdist
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --sdist --out-dir /tmp/sdist/
-
-      - name: Generate formula from local sdist
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)"
-          SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')"
-
-          # Install archivebox + poet into a temp venv to generate resource stanzas
-          python3 -m venv /tmp/poet-venv
-          source /tmp/poet-venv/bin/activate
-          pip install --quiet "$SDIST_PATH" homebrew-pypi-poet
-          echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
-          RESOURCES="$(poet archivebox)"
-          deactivate
-
-          # For CI: use file:// URL pointing to local sdist
-          # For release: this gets overridden with the PyPI URL
-          SDIST_URL="file://${SDIST_PATH}"
-
-          cat > /tmp/archivebox.rb << RUBY
-class Archivebox < Formula
-  include Language::Python::Virtualenv
-
-  desc "Self-hosted internet archiving solution"
-  homepage "https://github.com/ArchiveBox/ArchiveBox"
-  url "${SDIST_URL}"
-  sha256 "${SDIST_SHA256}"
-  license "MIT"
-
-  depends_on "python@3.13"
-
-${RESOURCES}
-
-  def install
-    virtualenv_install_with_resources
-  end
-
-  def post_install
-    data_dir = var/"archivebox"
-    data_dir.mkpath
-    ENV["DATA_DIR"] = data_dir.to_s
-    system bin/"archivebox", "init"
-  end
-
-  def caveats
-    <<~EOS
-      ArchiveBox data is stored in:
-        #{var}/archivebox
-
-      To start archiving, run:
-        cd #{var}/archivebox && archivebox add 'https://example.com'
-
-      To start the web UI:
-        cd #{var}/archivebox && archivebox server 0.0.0.0:8000
-    EOS
-  end
-
-  test do
-    assert_match version.to_s, shell_output("#{bin}/archivebox version")
-  end
-end
-RUBY
-
-          echo "[√] Generated formula:"
-          ruby -c /tmp/archivebox.rb
-          cat /tmp/archivebox.rb
-
-      - name: Install Homebrew (Linux only)
-        if: runner.os == 'Linux'
-        run: |
-          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
-          echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV"
-          echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV"
-          echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV"
-          echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH"
-          echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH"
-
-      - name: Install brew dependencies
-        run: brew install python@3.13
-
-      - name: Install archivebox via brew from local formula
-        run: |
-          brew install --build-from-source --verbose /tmp/archivebox.rb
-
-      - name: Verify archivebox version
-        run: archivebox version
-
-      - name: Test archivebox init
-        run: |
-          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
-          archivebox init
-
-      - name: Test archivebox status
-        run: |
-          cd /tmp/archivebox-test
-          archivebox status
-
-  # On release only: generate the real formula with PyPI URL and push to tap
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: build-and-test
-    runs-on: macos-latest
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Wait for PyPI package availability
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..."
-          for i in $(seq 1 30); do
-            if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then
-              echo "[√] archivebox==${VERSION} is available on PyPI"
-              break
-            fi
-            if [ "$i" -eq 30 ]; then
-              echo "[!] Timed out waiting for PyPI. Trying to install anyway..."
-              break
-            fi
-            echo "    Attempt $i/30 - not yet available, waiting 30s..."
-            sleep 30
-          done
-
-      - name: Generate release formula via build_brew.sh
-        run: ./bin/build_brew.sh
-
-      - name: Test formula install
-        run: |
-          brew install --build-from-source brew_dist/archivebox.rb
-          archivebox version
-
-      - name: Upload formula artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox.rb
-          path: brew_dist/archivebox.rb
-
-      - name: Push to homebrew-archivebox tap
-        env:
-          GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap
-
-          cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \
-            cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb
-
-          cd /tmp/tap
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add -A
-          if git diff --cached --quiet; then
-            echo "[i] No changes to formula, skipping push."
-          else
-            git commit -m "Update archivebox to v${VERSION}"
-            git push origin HEAD
-            echo "[√] Formula pushed to homebrew-archivebox tap"
-          fi

.github/workflows/lint.yml

@@ -4,6 +4,10 @@ on:
   workflow_dispatch:
   push:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   UV_NO_SOURCES: "1"
   PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
@@ -18,13 +22,13 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -35,12 +39,13 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - name: Install dependencies with uv
         run: |
-          uv sync --all-extras --all-groups --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --all-groups -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[sonic,debug]"
 
       - name: Run prek
         run: uv run --no-sync --no-sources prek run --all-files

.github/workflows/pip.yml

@@ -9,6 +9,10 @@ on:
     tags:
       - 'v*'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   PYTHON_VERSION: "3.13"
 
@@ -18,12 +22,14 @@ jobs:
       id-token: write
 
     runs-on: ubuntu-24.04
+    environment: pypi
     steps:
       - uses: actions/checkout@v4
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - name: Set up Python
@@ -38,8 +44,89 @@ jobs:
           packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
           version: 1.0
 
+      - name: Wait for released ArchiveBox deps on PyPI
+        run: |
+          python - <<'PY'
+          import re
+          import sys
+          import time
+          import tomllib
+          import urllib.request
+
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  try:
+                      with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20):
+                          available = True
+                  except Exception:
+                      available = False
+                  if available:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
+          PY
+
+      - name: Prepare released dependency sync project
+        run: |
+          python - <<'PY'
+          from pathlib import Path
+          import json
+          import re
+          import urllib.request
+
+          source = Path("pyproject.toml")
+          target_dir = Path(".ci-uv-project")
+          target_dir.mkdir(exist_ok=True)
+          text = source.read_text()
+          text = text.replace(
+              'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
+              'environments = ["sys_platform == \'linux\'"]',
+          )
+
+          # CI runs immediately after publishing internal abx packages. PyPI's
+          # simple index can lag the version JSON endpoint, so sync against the
+          # exact wheel URLs from the version endpoint while leaving package
+          # metadata in pyproject.toml unchanged for the actual build.
+          for package in ("abxpkg", "abx-plugins", "abx-dl"):
+              match = re.search(rf'"{re.escape(package)}>=(?P<version>[^"]+)"', text)
+              if not match:
+                  continue
+              version = match.group("version")
+              with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
+                  data = json.load(response)
+              wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
+              text = re.sub(
+                  rf'"{re.escape(package)}>=[^"]+"',
+                  f'"{package} @ {wheel_url}"',
+                  text,
+                  count=1,
+              )
+
+          (target_dir / "pyproject.toml").write_text(text)
+          PY
+
       - name: UV install archivebox dev + run sub-dependencies
-        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+        env:
+          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
+        run: uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
 
       - name: UV build archivebox and archivebox/pkgs/* packages
         run: |
@@ -49,17 +136,22 @@ jobs:
         uses: pypa/gh-action-pypi-publish@release/v1
         # ignore when publish to PyPI fails due to duplicate tag
         continue-on-error: true
-        with:
-          password: ${{ secrets.PYPI_PAT_SECRET }}
 
       - name: UV install archivebox and archivebox/pkgs/* locally for tests
-        run: uv sync --all-extras --no-sources --no-cache
-
-      - name: UV run archivebox init + archivebox version
+        env:
+          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
         run: |
-          mkdir -p data && cd data
-          uv run --no-sync --no-sources archivebox init \
-          && uv run --no-sync --no-sources archivebox version
-          # && uv run archivebox add 'https://example.com' \
-          # && uv run archivebox status \
-          # || (echo "UV Failed to run archivebox!" && exit 1)
+          uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+          uv pip install --python .venv/bin/python --no-deps dist/archivebox-*.whl
+
+      - name: Verify built package full install
+        env:
+          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
+        run: |
+          set -Eeuo pipefail
+          DATA_DIR="$(mktemp -d)"
+          cd "$DATA_DIR"
+          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox init
+          timeout 30m uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox install
+          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox version
+          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox status

.github/workflows/release-runner.yml

@@ -3,22 +3,30 @@ name: Release State
 on:
   push:
     branches:
-      - '**'
+      - dev
   workflow_dispatch:
 
 permissions:
+  actions: write
   contents: write
   id-token: write
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
 jobs:
   release-state:
+    if: github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'release:')
     runs-on: ubuntu-latest
+    environment: pypi
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: true
           ref: ${{ github.ref_name }}
+          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
 
       - uses: actions/setup-python@v5
         with:
@@ -26,6 +34,7 @@ jobs:
 
       - uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - uses: actions/setup-node@v4
@@ -39,7 +48,13 @@ jobs:
 
       - name: Run release script
         env:
-          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-          GH_TOKEN: ${{ github.token }}
-          PYPI_PAT_SECRET: ${{ secrets.PYPI_PAT_SECRET }}
+          DEFAULT_BRANCH: dev
+          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
         run: ./bin/release.sh
+
+      - name: Dispatch image and tap updates
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
+        run: |
+          gh workflow run docker.yml --ref dev
+          gh workflow run update-homebrew-tap.yml --ref dev

.github/workflows/release.yml

@@ -2,8 +2,8 @@ name: Release
 
 # Orchestrates the full release pipeline:
 #   1. Build and publish pip package to PyPI
-#   2. Build .deb packages and Homebrew formula (in parallel, after pip)
-#   3. Build Docker images (in parallel with deb/brew)
+#   2. Build Homebrew formula after pip
+#   3. Build Docker images in parallel
 #
 # Individual workflows also run on push for CI (see their own triggers).
 # Manual fallback for the hosted release pipeline. The normal prerelease loop
@@ -18,18 +18,16 @@ permissions:
   packages: write
   id-token: write
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   pip:
     name: Publish to PyPI
     uses: ./.github/workflows/pip.yml
     secrets: inherit
 
-  debian:
-    name: Build .deb packages
-    needs: pip
-    uses: ./.github/workflows/debian.yml
-    secrets: inherit
-
   homebrew:
     name: Update Homebrew formula
     needs: pip

.github/workflows/test-parallel.yml

@@ -1,21 +1,27 @@
 name: Parallel Tests
 
 on:
+  workflow_dispatch:
   pull_request:
     branches: [dev, main, master]
   push:
     branches: [dev]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   PYTHONIOENCODING: utf-8
   PYTHONLEGACYWINDOWSSTDIO: utf-8
   USE_COLOR: False
   UV_NO_SOURCES: "1"
+  ABXPKG_MIN_RELEASE_AGE: "0"
 
 jobs:
   discover-tests:
     name: Discover test files
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     outputs:
       test-files: ${{ steps.set-matrix.outputs.test-files }}
       plugin-tests: ${{ steps.set-plugin-matrix.outputs.plugin-tests }}
@@ -44,8 +50,16 @@ jobs:
 
             # Extract a display name for the test
             name="main/$(basename $test_file .py | sed 's/^test_//')"
+            needs_chromium=false
+            if grep -Eiq '(chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue)' "$test_file"; then
+              needs_chromium=true
+            fi
+            needs_sonic=false
+            if grep -Eiq "shutil\\.which\\([\"']sonic|SEARCH_BACKEND_ENGINE=.*sonic|worker_sonic" "$test_file"; then
+              needs_sonic=true
+            fi
 
-            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
           done
           json_array+="]"
 
@@ -54,7 +68,7 @@ jobs:
           echo "$json_array" | jq '.'
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Discover plugin tests
         id: set-plugin-matrix
@@ -74,8 +88,12 @@ jobs:
             if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
               needs_chromium=true
             fi
+            needs_sonic=false
+            if [ "$plugin_name" = "search_backend_sonic" ]; then
+              needs_sonic=true
+            fi
 
-            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium}"
+            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
           done
           json_array+="]"
 
@@ -85,7 +103,7 @@ jobs:
 
   run-tests:
     name: ${{ matrix.test.name }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: discover-tests
     env:
       PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
@@ -107,24 +125,24 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
           architecture: x64
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v6
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -132,7 +150,7 @@ jobs:
           node-version: 22
 
       - name: Cache uv
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/uv
           key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
@@ -146,17 +164,89 @@ jobs:
 
       - name: Install dependencies with uv
         run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
+
+      - name: Install Chrome through ArchiveBox
+        if: ${{ matrix.test.needs_chromium }}
+        env:
+          MIN_CHROMIUM_MAJOR: "149"
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
+          case "$major" in
+            ''|*[!0-9]*) major=0 ;;
+          esac
+          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
+            exit 1
+          fi
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
+          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "DATA_DIR=$ABX_CI_DATA_DIR" >> "$GITHUB_ENV"
+          echo "NODE_MODULES_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules" >> "$GITHUB_ENV"
+          echo "NODE_PATH=$LIB_DIR/pnpm/packages/chrome/node_modules" >> "$GITHUB_ENV"
+          echo "PNPM_BIN_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules/.bin" >> "$GITHUB_ENV"
+          echo "NPM_BIN_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules/.bin" >> "$GITHUB_ENV"
+          echo "CHROMEWEBSTORE_EXTENSIONS_DIR=$LIB_DIR/chromewebstore/extensions" >> "$GITHUB_ENV"
+
+      - name: Install Sonic through ArchiveBox
+        if: ${{ matrix.test.needs_sonic }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
+          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
 
       - name: Run test - ${{ matrix.test.name }}
         run: |
           mkdir -p tests/out
+          set +e
           uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
+          status=$?
+          set -e
+          if [ "$status" -eq 5 ]; then
+            echo "No tests collected from ${{ matrix.test.path }}; treating as an empty test module."
+            exit 0
+          fi
+          exit "$status"
 
   plugin-tests:
     name: ${{ matrix.plugin.name }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: discover-tests
     env:
       PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
@@ -177,24 +267,24 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       - name: Set up Python 3.13
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.13"
           architecture: x64
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v6
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -202,7 +292,7 @@ jobs:
           node-version: 22
 
       - name: Cache uv
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/uv
           key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }}
@@ -216,58 +306,28 @@ jobs:
 
       - name: Install dependencies with uv
         run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
 
-      - name: Select Chromium for extension-capable CDP
+      - name: Install Chrome through ArchiveBox
         if: ${{ matrix.plugin.needs_chromium }}
         env:
           MIN_CHROMIUM_MAJOR: "149"
           LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
         run: |
           set -euo pipefail
-          mkdir -p "$LIB_DIR"
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
           echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
           echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
 
-          candidate="/usr/bin/chromium"
-          version=""
-          major=0
-          if [ -x "$candidate" ]; then
-            version="$("$candidate" --version || true)"
-            major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
-            case "$major" in
-              ''|*[!0-9]*) major=0 ;;
-            esac
-          fi
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
 
-          if [ "$major" -ge "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Using runner Chromium: $candidate ($version)"
-            echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
-            exit 0
-          fi
-
-          echo "Runner Chromium is unavailable or too old: ${version:-not found}"
-          echo "Installing Playwright Chromium into $LIB_DIR..."
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=npm \
-            --lib="$LIB_DIR" \
-            --postinstall-scripts=false \
-            --min-release-age=0 \
-            --no-cache \
-            --overrides='{"npm":{"install_args":["playwright@next"]}}' \
-            playwright
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=playwright \
-            --lib="$LIB_DIR" \
-            --bin-dir="$LIB_DIR/env/bin" \
-            --min-version="$MIN_CHROMIUM_MAJOR" \
-            chromium
-
-          candidate="$LIB_DIR/env/bin/chromium"
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
           if [ ! -x "$candidate" ]; then
-            echo "Playwright Chromium shim was not created at $candidate" >&2
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
             exit 1
           fi
           version="$("$candidate" --version || true)"
@@ -276,11 +336,42 @@ jobs:
             ''|*[!0-9]*) major=0 ;;
           esac
           if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Playwright Chromium is still too old: ${version:-unknown}" >&2
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
             exit 1
           fi
-          echo "Using Playwright Chromium: $candidate ($version)"
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
           echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "DATA_DIR=$ABX_CI_DATA_DIR" >> "$GITHUB_ENV"
+          echo "NODE_MODULES_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules" >> "$GITHUB_ENV"
+          echo "NODE_PATH=$LIB_DIR/pnpm/packages/chrome/node_modules" >> "$GITHUB_ENV"
+          echo "PNPM_BIN_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules/.bin" >> "$GITHUB_ENV"
+          echo "NPM_BIN_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules/.bin" >> "$GITHUB_ENV"
+          echo "CHROMEWEBSTORE_EXTENSIONS_DIR=$LIB_DIR/chromewebstore/extensions" >> "$GITHUB_ENV"
+
+      - name: Install Sonic through ArchiveBox
+        if: ${{ matrix.plugin.needs_sonic }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
+          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
 
       - name: Run plugin tests - ${{ matrix.plugin.name }}
         env:

.github/workflows/test.yml

@@ -1,6 +1,10 @@
 name: Run tests
 on: [push]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   DOCKER_IMAGE: archivebox-ci
   PYTHONIOENCODING: utf-8
@@ -27,13 +31,13 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       ### Setup Python & JS Languages
       - name: Set up Python ${{ matrix.python }}
@@ -45,7 +49,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v4
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -68,8 +72,8 @@ jobs:
 
       - name: Install dependencies with uv
         run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
 
       ### Run the tests
       - name: Directory listing for debugging
@@ -77,10 +81,15 @@ jobs:
           pwd
           ls
 
-      - name: Archivebox version
+      - name: ArchiveBox full install check
         run: |
-          mkdir -p tests/out/data
-          DATA_DIR="$PWD/tests/out/data" uv run --no-sync --no-sources archivebox version
+          set -Eeuo pipefail
+          DATA_DIR="$PWD/tests/out/data"
+          mkdir -p "$DATA_DIR"
+          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox init
+          timeout 30m uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox install
+          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox version
+          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox status
 
       - name: Test built package with pytest
         # TODO: remove this exception for windows once we get tests passing on that platform

.github/workflows/update-homebrew-tap.yml

@@ -0,0 +1,37 @@
+name: Update downstream packages
+
+on:
+  push:
+    branches: [dev]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: update-downstream-packages-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dispatch:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Dispatch downstream package updates
+        env:
+          DOWNSTREAM_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || secrets.HOMEBREW_TAP_TOKEN }}
+        run: |
+          if [ -z "$DOWNSTREAM_TOKEN" ]; then
+            echo "RELEASE_GH_TOKEN/HOMEBREW_TAP_TOKEN is not configured; downstream schedules will update packages."
+            exit 0
+          fi
+
+          for repo in homebrew-archivebox debian-archivebox; do
+            curl --fail-with-body \
+              -X POST \
+              -H "Accept: application/vnd.github+json" \
+              -H "Authorization: Bearer ${DOWNSTREAM_TOKEN}" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              "https://api.github.com/repos/ArchiveBox/${repo}/dispatches" \
+              -d '{"event_type":"archivebox-dev-updated","client_payload":{"ref":"dev"}}'
+          done

.gitignore

@@ -63,3 +63,6 @@ ArchiveBox.conf
 # vim
 *.sw?
 .vscode
+
+# Local Claude Code task locks (not project state)
+.claude/scheduled_tasks.lock

.pre-commit-config.yaml

@@ -29,12 +29,12 @@ repos:
     hooks:
       - id: ruff-format
         name: ruff-format
-        entry: uv run --active --no-sources ruff format
+        entry: uv run --active --no-sync --no-sources ruff format
         language: system
         types_or: [python, pyi]
       - id: ruff-check
         name: ruff-check
-        entry: uv run --active --no-sources ruff check --fix
+        entry: uv run --active --no-sync --no-sources ruff check --fix
         language: system
         types_or: [python, pyi]
 

AGENTS.md

@@ -0,0 +1,87 @@
+# ArchiveBox Agent Guide
+
+ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev` branch.
+
+## Shared Standards
+
+- Use `uv` and `uv run` for Python commands. Do not use system `python`, direct `.venv/bin/python`, or `pip` commands.
+- Prefer existing repo patterns, helper APIs, fixtures, scripts, and command surfaces.
+- Keep edits focused and minimal. Do not add wrappers, shims, aliases, or extra abstraction layers unless the current code path requires them.
+- Do not weaken assertions, skip tests, xfail tests, or accept flaky behavior.
+- No mocks, monkeypatches, fakes, simulated handlers, fake binaries, fake hooks, fake buses, or direct shortcuts around user-facing flows.
+- Tests and verification should use real CLI commands, REST/API calls, browser UI flows, real hooks, real installs, real subprocesses, real DB rows, real files, and existing fixtures.
+- Assertions must verify real correctness: exit codes, returned values, DB state, filesystem contents, field values, rendered output, and side effects.
+- Start behavior fixes with a red failing test when a test is requested or practical.
+- Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
+- Read `README.md` for the full setup, CLI, Docker, API, and release surface.
+
+## Development Setup
+
+```bash
+uv sync --dev --all-extras
+mkdir -p data
+cd data
+uv run --project .. archivebox init --install
+```
+
+Run collection commands from inside an initialized data directory:
+
+```bash
+cd data
+uv run --project .. archivebox status
+uv run --project .. archivebox add 'https://example.com'
+uv run --project .. archivebox run
+```
+
+## User-Facing Setup
+
+Recommended CLI install:
+
+```bash
+uv tool install archivebox
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init --install
+archivebox add 'https://example.com'
+archivebox server 0.0.0.0:8000
+```
+
+Alternative install methods:
+
+- Docker Compose / Docker
+- Homebrew
+- Debian package
+- pip
+
+## Basic Usage
+
+```bash
+archivebox version
+archivebox help
+archivebox status
+archivebox install
+archivebox add 'https://example.com'
+archivebox add --extract=title,screenshot,pdf 'https://example.com'
+archivebox list --json --with-headers
+archivebox search 'example'
+archivebox update --filter-type=domain example.com
+archivebox remove --filter-type=exact 'https://example.com'
+archivebox run
+```
+
+## Verification
+
+Use targeted tests for focused work:
+
+<!--pytest.mark.skip(reason="pytest invocation")-->
+```bash
+uv run pytest archivebox/tests/test_cli_add.py -q
+uv run prek run --all-files
+```
+
+Use the full release/deploy loop only when requested:
+
+<!--pytest.mark.skip(reason="release/deploy script")-->
+```bash
+./bin/release_dev_stack.sh
+```

CLAUDE.md

@@ -1,497 +0,0 @@
-# Claude Code Development Guide for ArchiveBox
-
-## Quick Start
-
-```bash
-# Set up dev environment (always use uv, never pip directly)
-uv sync --dev --all-extras
-
-# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
-```
-
-## Development Environment Setup
-
-### Prerequisites
-- Python 3.11+ (3.13 recommended)
-- uv package manager
-- A non-root user for running tests (e.g., `testuser`)
-
-### Install Dependencies
-```bash
-uv sync --dev --all-extras  # Always use uv, never pip directly
-```
-
-### Activate Virtual Environment
-```bash
-source .venv/bin/activate
-```
-
-### Common Gotchas
-
-#### File Permissions
-New files created by root need permissions fixed for testuser:
-```bash
-chmod 644 archivebox/tests/test_*.py
-```
-
-#### DATA_DIR Environment Variable
-ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
-
-## Code Style Guidelines
-
-### Naming Conventions for Grep-ability
-Use consistent naming for everything to enable easy grep-ability and logical grouping:
-
-**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
-
-**Examples**:
-```python
-# Filesystem migration methods - all start with fs_
-def fs_migration_needed() -> bool: ...
-def fs_migrate() -> None: ...
-def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
-def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
-def _fs_next_version(current: str) -> str: ...
-
-# Logging methods - ALL must start with log_ or _log
-def log_migration_start(snapshot_id: str) -> None: ...
-def _log_error(message: str) -> None: ...
-def log_validation_result(ok: bool, msg: str) -> None: ...
-```
-
-**Rules**:
-- Group related functions with common prefixes
-- Use `_` prefix for internal/private helpers within the same family
-- ALL logging-related methods MUST start with `log_` or `_log`
-- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
-- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
-
-### Minimize Unique Names and Data Structures
-**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
-
-**Example - GOOD**:
-```python
-# Binary has overrides field
-binary = Binary(overrides={'TIMEOUT': '60s'})
-
-# Binary reuses the same field name and structure
-class Binary(models.Model):
-    overrides = models.JSONField(default=dict)  # Same name, same structure
-```
-
-**Example - BAD**:
-```python
-# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
-class Binary(models.Model):
-    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
-```
-
-**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
-
-## Testing
-
-### CRITICAL: Never Run as Root
-ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
-
-```bash
-# Run all migration tests
-sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
-
-# Run specific test file
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
-
-# Run single test
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
-```
-
-### Test File Structure
-```
-archivebox/tests/
-├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
-├── test_migrations_fresh.py      # Fresh install tests
-├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
-├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
-└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
-```
-
-### Test Writing Standards
-
-#### NO MOCKS - Real Tests Only
-Tests must exercise real code paths:
-- Create real SQLite databases with version-specific schemas
-- Seed with realistic test data
-- Run actual `python -m archivebox` commands via subprocess
-- Query SQLite directly to verify results
-
-**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
-
-#### NO SKIPS
-Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
-
-#### Strict Assertions
-- `init` command must return exit code 0 (not `[0, 1]`)
-- Verify ALL data is preserved, not just "at least one"
-- Use exact counts (`==`) not loose bounds (`>=`)
-
-### Example Test Pattern
-```python
-def test_migration_preserves_snapshots(self):
-    """Migration should preserve all snapshots."""
-    result = run_archivebox(self.work_dir, ['init'], timeout=45)
-    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
-
-    ok, msg = verify_snapshot_count(self.db_path, expected_count)
-    self.assertTrue(ok, msg)
-```
-
-### Testing Gotchas
-
-#### Extractors Disabled for Speed
-Tests disable all extractors via environment variables for faster execution:
-```python
-env['SAVE_TITLE'] = 'False'
-env['SAVE_FAVICON'] = 'False'
-# ... etc
-```
-
-#### Timeout Settings
-Use appropriate timeouts for migration tests (45s for init, 60s default).
-
-### Plugin Testing & Code Coverage
-
-**Target: 80-90% coverage** for critical plugins (screenshot, chrome, singlefile, dom)
-
-```bash
-# Run plugin tests with coverage (both Python + JavaScript)
-bash bin/test_plugins.sh screenshot
-
-# View coverage reports
-bash bin/test_plugins.sh --coverage-report
-# Or individual reports:
-coverage report --show-missing --include='archivebox/plugins/*' --omit='*/tests/*'
-```
-
-#### Plugin Test Structure
-
-Tests are **completely isolated** from ArchiveBox - they replicate production directory structure in temp dirs:
-
-```python
-# Correct production paths:
-# Crawl:    DATA_DIR/users/{username}/crawls/YYYYMMDD/example.com/{crawl-id}/{plugin}/
-# Snapshot: DATA_DIR/users/{username}/snapshots/YYYYMMDD/example.com/{snapshot-uuid}/{plugin}/
-
-with tempfile.TemporaryDirectory() as tmpdir:
-    data_dir = Path(tmpdir)
-
-    # Crawl-level plugin (e.g., chrome launcher)
-    crawl_dir = data_dir / 'users' / 'testuser' / 'crawls' / '20240101' / 'example.com' / 'crawl-123'
-    chrome_dir = crawl_dir / 'chrome'
-    chrome_dir.mkdir(parents=True)
-
-    # Snapshot-level plugin (e.g., screenshot)
-    snapshot_dir = data_dir / 'users' / 'testuser' / 'snapshots' / '20240101' / 'example.com' / 'snap-456'
-    screenshot_dir = snapshot_dir / 'screenshot'
-    screenshot_dir.mkdir(parents=True)
-
-    # Run hook in its output directory
-    result = subprocess.run(
-        ['node', str(SCREENSHOT_HOOK), '--url=https://example.com'],
-        cwd=str(screenshot_dir),
-        env={**get_test_env(), 'EXTRA_CONTEXT': '{"snapshot_id":"snap-456"}'},
-        capture_output=True,
-        timeout=120
-    )
-```
-
-#### Coverage Improvement Loop
-
-To improve from ~20% to 80%+:
-
-1. **Run tests**: `bash bin/test_plugins.sh screenshot` → Shows: `19.1% (13/68 ranges)`
-2. **Identify gaps**: Check hook file for untested paths (session connection vs fallback, config branches, error cases)
-3. **Add tests**: Test both execution paths (connect to session + launch own browser), skip conditions, error cases, config variations
-4. **Verify**: Re-run tests → Should show: `85%+ (58+/68 ranges)`
-
-**Critical**: JavaScript hooks have TWO paths that both must be tested (connect to session ~50% + launch browser ~30% + shared ~20%). Testing only one path = max 50% coverage possible!
-
-## Database Migrations
-
-### Generate and Apply Migrations
-```bash
-# Generate migrations (run from archivebox subdirectory)
-cd archivebox
-./manage.py makemigrations
-
-# Apply migrations to test database
-cd data/
-archivebox init
-```
-
-### Schema Versions
-- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
-- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
-- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
-- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
-
-### Testing a Migration Path
-1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
-2. Seed with realistic test data using `seed_0_X_data()`
-3. Run `archivebox init` to trigger migrations
-4. Verify data preservation with `verify_*` functions
-5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
-
-### Squashed Migrations
-When testing 0.8.x (dev branch), you must record ALL replaced migrations:
-```python
-# The squashed migration replaces these - all must be recorded
-('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
-('core', '0024_auto_20240513_1143'),
-# ... all 52 migrations from 0023-0074 ...
-('core', '0023_new_schema'),  # Also record the squashed migration itself
-```
-
-### Migration Strategy
-- Squashed migrations for clean installs
-- Individual migrations recorded for upgrades from dev branch
-- `replaces` attribute in squashed migrations lists what they replace
-
-### Migration Gotchas
-
-#### Circular FK References in Schemas
-SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
-
-## Plugin System Architecture
-
-### Plugin Dependency Rules
-
-Like other plugins, chrome plugins **ARE NOT ALLOWED TO DEPEND ON ARCHIVEBOX OR DJANGO**.
-However, they are allowed to depend on two shared files ONLY:
-- `archivebox/plugins/chrome/chrome_utils.js` ← source of truth API for all basic chrome ops
-- `archivebox/plugins/chrome/tests/chrome_test_utils.py` ← use for your tests, do not implement launching/killing/pid files/cdp/etc. in python, just extend this file as needed.
-
-### Chrome-Dependent Plugins
-
-Many plugins depend on Chrome/Chromium via CDP (Chrome DevTools Protocol). When checking for script name references or debugging Chrome-related issues, check these plugins:
-
-**Main puppeteer-based chrome installer + launcher plugin**:
-- `chrome` - Core Chrome integration (CDP, launch, navigation)
-
-**Metadata extraction using chrome/chrome_utils.js / CDP**:
-- `dns` - DNS resolution info
-- `ssl` - SSL certificate info
-- `headers` - HTTP response headers
-- `redirects` - Capture redirect chains
-- `staticfile` - Direct file downloads (e.g. if the url itself is a .png, .exe, .zip, etc.)
-- `responses` - Capture network responses
-- `consolelog` - Capture console.log output
-- `title` - Extract page title
-- `accessibility` - Extract accessibility tree
-- `seo` - Extract SEO metadata
-
-**Extensions installed using chrome/chrome_utils.js / controlled using CDP**:
-- `ublock` - uBlock Origin ad blocking
-- `istilldontcareaboutcookies` - Cookie banner dismissal
-- `twocaptcha` - 2captcha CAPTCHA solver integration
-
-**Page-alteration plugins to prepare the content for archiving**:
-- `modalcloser` - Modal dialog dismissal
-- `infiniscroll` - Infinite scroll handler
-
-**Main Extractor Outputs**:
-- `dom` - DOM snapshot extraction
-- `pdf` - Generate PDF snapshots
-- `screenshot` - Generate screenshots
-- `singlefile` - SingleFile archival, can be single-file-cli that launches chrome, or singlefile extension running inside chrome
-
-**Crawl URL parsers** (post-process dom.html, singlefile.html, staticfile, responses, headers, etc. for URLs to re-emit as new queued Snapshots during recursive crawling):
-- `parse_dom_outlinks` - Extract outlinks from DOM (special, uses CDP to directly query browser)
-- `parse_html_urls` - Parse URLs from HTML (doesn't use chrome directly, just reads dom.html)
-- `parse_jsonl_urls` - Parse URLs from JSONL (doesn't use chrome directly, just reads dom.html)
-- `parse_netscape_urls` - Parse Netscape bookmark format (doesn't use chrome directly, just reads dom.html)
-
-### Finding Chrome-Dependent Plugins
-
-```bash
-# Find all files containing "chrom" (case-insensitive)
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d: -f1 | sort -u
-
-# Or get just the plugin names
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d/ -f3 | sort -u
-```
-
-**Note**: This list may not be complete. Always run the grep command above when checking for Chrome-related script references or debugging Chrome integration issues.
-
-## Architecture Notes
-
-### Crawl Model (0.9.x)
-- Crawl groups multiple Snapshots from a single `add` command
-- Each `add` creates one Crawl with one or more Snapshots
-- Seed model was removed - crawls now store URLs directly
-
-## Code Coverage
-
-### Overview
-
-Coverage tracking is enabled for passive collection across all contexts:
-- Unit tests (pytest)
-- Integration tests
-- Dev server (manual testing)
-- CLI usage
-
-Coverage data accumulates in `.coverage` file and can be viewed/analyzed to find dead code.
-
-### Install Coverage Tools
-
-```bash
-uv sync --dev  # Installs pytest-cov and coverage
-```
-
-### Running with Coverage
-
-#### Unit Tests
-```bash
-# Run tests with coverage
-pytest --cov=archivebox --cov-report=term archivebox/tests/
-
-# Or run specific test file
-pytest --cov=archivebox --cov-report=term archivebox/tests/test_migrations_08_to_09.py
-```
-
-#### Dev Server with Coverage
-```bash
-# Start dev server with coverage tracking
-coverage run --parallel-mode -m archivebox server
-
-# Or CLI commands
-coverage run --parallel-mode -m archivebox init
-coverage run --parallel-mode -m archivebox add https://example.com
-```
-
-#### Manual Testing (Always-On)
-To enable coverage during ALL Python executions (passive tracking):
-
-```bash
-# Option 1: Use coverage run wrapper
-coverage run --parallel-mode -m archivebox [command]
-
-# Option 2: Set environment variable (tracks everything)
-export COVERAGE_PROCESS_START=pyproject.toml
-# Now all Python processes will track coverage
-archivebox server
-archivebox add https://example.com
-```
-
-### Viewing Coverage
-
-#### Text Report (Quick View)
-```bash
-# Combine all parallel coverage data
-coverage combine
-
-# View summary
-coverage report
-
-# View detailed report with missing lines
-coverage report --show-missing
-
-# View specific file
-coverage report --include="archivebox/core/models.py" --show-missing
-```
-
-#### JSON Report (LLM-Friendly)
-```bash
-# Generate JSON report
-coverage json
-
-# View the JSON
-cat coverage.json | jq '.files | keys'  # List all files
-
-# Find files with low coverage
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 50) | "\(.key): \(.value.summary.percent_covered)%"'
-
-# Find completely uncovered files (dead code candidates)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# Get missing lines for a specific file
-cat coverage.json | jq '.files["archivebox/core/models.py"].missing_lines'
-```
-
-#### HTML Report (Visual)
-```bash
-# Generate interactive HTML report
-coverage html
-
-# Open in browser
-open htmlcov/index.html
-```
-
-### Isolated Runs
-
-To measure coverage for specific scenarios:
-
-```bash
-# 1. Reset coverage data
-coverage erase
-
-# 2. Run your isolated test/scenario
-pytest --cov=archivebox archivebox/tests/test_migrations_fresh.py
-# OR
-coverage run --parallel-mode -m archivebox add https://example.com
-
-# 3. View results
-coverage combine
-coverage report --show-missing
-
-# 4. Optionally export for analysis
-coverage json
-```
-
-### Finding Dead Code
-
-```bash
-# 1. Run comprehensive tests + manual testing to build coverage
-pytest --cov=archivebox archivebox/tests/
-coverage run --parallel-mode -m archivebox server  # Use the app manually
-coverage combine
-
-# 2. Find files with 0% coverage (strong dead code candidates)
-coverage json
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# 3. Find files with <10% coverage (likely dead code)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 10) | "\(.key): \(.value.summary.percent_covered)%"' | sort -t: -k2 -n
-
-# 4. Generate detailed report for analysis
-coverage report --show-missing > coverage_report.txt
-```
-
-### Tips
-
-- **Parallel mode** (`--parallel-mode`): Allows multiple processes to track coverage simultaneously without conflicts
-- **Combine**: Always run `coverage combine` before viewing reports to merge parallel data
-- **Reset**: Use `coverage erase` to start fresh for isolated measurements
-- **Branch coverage**: Enabled by default - tracks if both branches of if/else are executed
-- **Exclude patterns**: Config in `pyproject.toml` excludes tests, migrations, type stubs
-
-## Debugging Tips
-
-### Check Migration State
-```bash
-sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
-```
-
-### Check Table Schema
-```bash
-sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
-```
-
-### Verbose Test Output
-```bash
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
-```
-
-### Kill Zombie Chrome Processes
-```bash
-./bin/kill_chrome.sh
-```

Dockerfile

@@ -1,38 +1,188 @@
-# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
-#     python3.13, uv, python3-ldap
-#     curl, wget, git, dig, ping, tree, nano
-#     node, npm
-#     ArchiveBox and plugin runtime dependencies installed by archivebox init --install
-# Usage:
-#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
-#     docker build . -t archivebox
-#     docker run -v "$PWD/data":/data archivebox init
-#     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
-#     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
-#     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
-#     docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
-# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+# syntax=docker/dockerfile:1.7
 
+# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
+# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
+# ArchiveBox owns sonic, supervisor, Django, and the app runtime.
+# Build abx-dl first, then point this file at it:
+#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
+#       --build-context abxbus=../abxbus \
+#       --build-context abxpkg=../abxpkg \
+#       --build-context abx-plugins=../abx-plugins \
+#       -t archivebox/abx-dl:dev
+#   docker buildx build . -f Dockerfile \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
+#       -t archivebox:multistage
 
-#########################################################################################
+ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
 
-### Example: Using ArchiveBox in your own project's Dockerfile ########
-
-# FROM python:3.13-slim
-# WORKDIR /data
-# RUN pip install archivebox>=0.9.0   # use latest release here
-# RUN archivebox install
-# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
-
-#########################################################################################
+FROM archivebox/sonic:1.4.9 AS sonic
+FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
 
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
-ARG TARGETVARIANT=
+ARG TARGETVARIANT
+ARG ARCHIVEBOX_COMMIT_HASH=""
 
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ubuntu:24.04
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_COMPILE=1 \
+    PIP_ONLY_BINARY=aiohttp \
+    npm_config_loglevel=error
+
+ENV PYTHON_VERSION=3.13 \
+    NODE_VERSION=24
+
+ENV ARCHIVEBOX_USER=archivebox \
+    DEFAULT_ARCHIVEBOX_UID=911 \
+    DEFAULT_ARCHIVEBOX_GID=911 \
+    IN_DOCKER=True
+
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    CONFIG_DIR=/opt/archivebox \
+    ABXPKG_LIB_DIR=/opt/archivebox/lib \
+    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
+    PERSONAS_DIR=/data/personas \
+    CHROME_HEADLESS=true \
+    CHROME_SANDBOX=false \
+    CHROME_ISOLATION=crawl \
+    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
+
+ENV TMP_DIR=/tmp/archivebox \
+    PIP_VENV_PYTHON=/venv/bin/python3 \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no
+
+ENV HOME=/home/archivebox \
+    XDG_CONFIG_HOME=/home/archivebox/.config \
+    XDG_CACHE_HOME=/opt/archivebox/lib/cache \
+    ABXPKG_INSTALL_TIMEOUT=600 \
+    ABXPKG_POSTINSTALL_SCRIPTS=True \
+    ABXPKG_MIN_RELEASE_AGE=0 \
+    TIMEOUT=600
+
+ENV UV_COMPILE_BYTECODE=false \
+    UV_PYTHON_PREFERENCE=managed \
+    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv \
+    VIRTUAL_ENV=/venv \
+    PATH="/venv/bin:/opt/node/bin:$PATH"
+
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
+WORKDIR "$CODE_DIR"
+
+RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
+    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && uname -a \
+    && sed -n '1,7p' /etc/os-release \
+    && abxpkg load --binproviders=env node \
+    && abxpkg load --binproviders=env uv \
+    ) | tee -a /VERSION.txt
+
+FROM archivebox-runtime-base AS archivebox-builder
+
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    <<'EOF'
+echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
+echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache
+echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends
+echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests
+rm -f /etc/apt/apt.conf.d/docker-clean
+apt-get update -qq
+apt-get install -qq -y --no-install-recommends \
+    build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+/usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}"
+/usr/bin/uv pip install setuptools pip wheel
+
+mkdir -p /tmp/archivebox-uv-project
+/venv/bin/python - <<'PY'
+from pathlib import Path
+import json
+import re
+import urllib.request
+
+source = Path("/app/pyproject.toml")
+target = Path("/tmp/archivebox-uv-project/pyproject.toml")
+text = source.read_text()
+text = text.replace(
+    'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
+    'environments = ["sys_platform == \'linux\'"]',
+)
+
+# Docker builds need the just-published internal abx wheels immediately, but
+# PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a
+# Docker-only dependency view from the version JSON so the published package
+# metadata stays normal while image builds remain resumable after a release.
+for package in ("abxpkg", "abx-plugins", "abx-dl"):
+    match = re.search(rf'"{re.escape(package)}>=(?P<version>[^"]+)"', text)
+    if not match:
+        continue
+    version = match.group("version")
+    with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
+        data = json.load(response)
+    wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
+    text = re.sub(
+        rf'"{re.escape(package)}>=[^"]+"',
+        f'"{package} @ {wheel_url}"',
+        text,
+        count=1,
+    )
+
+target.write_text(text)
+PY
+
+/usr/bin/uv sync \
+    --project /tmp/archivebox-uv-project \
+    --refresh \
+    --no-dev \
+    --inexact \
+    --no-install-project \
+    --no-install-workspace \
+    --no-sources
+(find /venv/lib/python3.*/site-packages -type f -name '*.so' -exec strip --strip-unneeded {} + 2>/dev/null || true)
+rm -f /venv/bin/uv /venv/bin/uvx
+apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+apt-get autoremove -y
+rm -rf /var/lib/apt/lists/*
+EOF
+
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && COMMIT_HASH="$( \
+        if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+            echo "$ARCHIVEBOX_COMMIT_HASH"; \
+        elif [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
+            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
+            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+                echo "$HEAD_REF"; \
+            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
+                REF_PATH="${HEAD_REF#ref: }"; \
+                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
+            fi; \
+        fi)" \
+    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
+    && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
+    && rm -f /venv/bin/uv /venv/bin/uvx \
+    && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
+
+FROM archivebox-runtime-base
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -43,334 +193,70 @@ LABEL name="archivebox" \
     org.opencontainers.image.vendor="ArchiveBox" \
     org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
     org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile" \
-    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
-    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
-    com.docker.desktop.extension.api.version=">= 1.4.7" \
-    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
-    com.docker.extension.publisher-url="https://archivebox.io" \
-    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
-    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
-    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
-    com.docker.extension.categories='database,utility-tools'
+    com.docker.image.source.entrypoint="Dockerfile"
 
-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-######### Environment Variables #################################
-
-# Global build-time and runtime environment constants + default pkg manager config
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    npm_config_loglevel=error
-
-# Language Version config
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22.22.3
-
-# Non-root User config
-ENV ARCHIVEBOX_USER="archivebox" \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
-
-# ArchiveBox Source Code + Lib + Data paths
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
-    LIB_BIN_DIR=/opt/archivebox/lib/bin \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/browsers
-
-# Bash SHELL config
-# http://redsymbol.net/articles/unofficial-bash-strict-mode/
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
-
-######### System Environment ####################################
-
-# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
-WORKDIR "$CODE_DIR"
-
-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-# Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && echo \
-    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
-    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
-    && echo \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && which bash && bash --version | sed -n '1p' \
-    && which dpkg && dpkg --version | sed -n '1p' \
-    && echo -e '\n\n' && env && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-# Create non-privileged user for archivebox and chrome
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system $ARCHIVEBOX_USER \
-    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
-    | tee -a /VERSION.txt
-    # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
-    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
-
-# Install system apt dependencies (adding backports to access more recent apt updates)
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
-    && mkdir -p /etc/apt/keyrings \
-    && apt-get update -qq \
-    && apt-get install -qq -y \
-        # 1. packaging dependencies
-        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
-        # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep dnsutils git python3.12-venv \
-        # 3. frivolous CLI helpers to make debugging failed archiving easier
-        tree nano iputils-ping \
-        # nano iputils-ping dnsutils htop procps jq yq
-    && rm -rf /var/lib/apt/lists/*
-
-# Install sonic search backend
 COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-RUN (which sonic && sonic --version) | tee -a /VERSION.txt
 
-######### Language Environments ####################################
+COPY --from=archivebox-builder /opt/uv/python /opt/uv/python
+COPY --from=archivebox-builder /venv /venv
+COPY --from=archivebox-builder /app /app
+COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
 
-# Set up Python environment
-# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
-#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
-    # && apt-get update -qq \
-    # && apt-get install -qq -y --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
-    # && rm -rf /var/lib/apt/lists/* \
-    # tell PDM to allow using global system python site packages
-    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
-    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
-    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
-    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
-    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
-    # && python3 -m venv $GLOBAL_VENV \
-    # install global dependencies / python build dependencies in GLOBAL_VENV
-    # && pip install --upgrade pip setuptools wheel \
-    # Save version info
-    # && ( \
-    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
-    #     && which pip && pip --version \
-    #     # && which pdm && pdm --version \
-    #     && echo -e '\n\n' \
-    # ) | tee -a /VERSION.txt
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
+    && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
+    && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
+    && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
+    && ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
+    && ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
+    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
+    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
+    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
+    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
+    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
+    && abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
+    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
+    && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$ABXPKG_LIB_DIR" \
+    && openssl rand -hex 16 > /etc/machine-id \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
+    && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
 
-
-# Set up Node environment from the official platform tarball. This avoids
-# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into
-# emulated Docker builds.
-RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    case "$TARGETARCH" in \
-        amd64) NODE_DIST_ARCH="x64" ;; \
-        arm64) NODE_DIST_ARCH="arm64" ;; \
-        *) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \
-    esac \
-    && NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \
-    && echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \
-    && grep "  ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \
-    && tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \
-    && rm "$NODE_TARBALL" SHASUMS256.txt \
-    # Save version info
-    && ( \
-        which node && node --version \
-        && which npm && npm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-# Set up uv and main app /venv
-RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
-ENV UV_COMPILE_BYTECODE=0 \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv
-WORKDIR "$CODE_DIR"
-# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
-    && uv venv /venv --python ${PYTHON_VERSION}
-ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
-RUN uv pip install setuptools pip \
-    && ( \
-        which python3 && python3 --version \
-        && which uv && uv self version \
-        && uv python find --system && uv python find \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-######### ArchiveBox & Extractor Dependencies ##################################
-
-# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc \
-        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
-        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-    && uv pip install \
-        "python-ldap>=3.4.3" \
-    && apt-get purge -y \
-        python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
-
-
-# Runtime config used by plugin hooks. Plugin binaries and npm packages are
-# installed into LIB_DIR below by archivebox init --install and resolved from
-# LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH.
-ENV PERSONAS_DIR=/data/personas \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-######### Build Dependencies ####################################
-
-
-# Install ArchiveBox Python venv dependencies from pyproject.toml.
-RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends build-essential gcc python3-dev \
-    && uv --no-cache sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /var/lib/apt/lists/*
-    # installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies
-
-# Setup ArchiveBox runtime config
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/usr/bin/python3.12 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-WORKDIR "$DATA_DIR"
-RUN openssl rand -hex 16 > /etc/machine-id \
-    && mkdir -p "$DATA_DIR" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" \
-    && mkdir -p "$TMP_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
-    && mkdir -p "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-# Pre-bake plugin-managed runtime dependencies using the same abx-dl installer
-# path users run later, before copying ArchiveBox source so source-only edits do
-# not invalidate the heavy browser/plugin dependency layer.
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
-    && apt-get update -qq \
-    && if [ "$TARGETARCH" = "arm64" ]; then \
-        abxpkg install --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
-    fi \
-    && PUID=0 PGID=0 abx-dl plugins --install \
-    && find "$LIB_DIR" "$DATA_DIR"/personas -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$LIB_DIR" "$DATA_DIR"/personas -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /var/lib/apt/lists/* \
-    && (chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR"/personas 2>/dev/null || true) \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"
-
-# Install ArchiveBox Python package from the checked-out source.
-WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && pip install \
-        --no-deps \
-        "$CODE_DIR" \
-    && ( \
-        pip show archivebox \
-        && which archivebox \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
-    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
-
-# Initialize an empty image collection without rerunning dependency installs.
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
-    && PUID=0 PGID=0 archivebox init \
-    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
+    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
+    && HOME="$TMP_DIR" archivebox init \
+    && (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
         "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
         "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
         "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
         "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        2>/dev/null || true)
+        "$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
+        "$TMP_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
+        2>/dev/null || true) \
+    && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
 
-# Print version for nice docker finish summary
-RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+RUN chmod +x "$CODE_DIR"/bin/*.sh \
+    && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+
+RUN --mount=type=cache,target=/tmp/abxpkg-cache,sharing=locked,mode=1777 \
+    for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
+    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
+    && python3 -c 'from abx_dl.models import discover_plugins; [print(f"export {plugin.enabled_key}=True") for plugin in discover_plugins(runtime="archivebox").values() if plugin.enabled_key in plugin.config.properties]' > /tmp/archivebox-enable-plugins.env \
+    && sort /tmp/archivebox-enable-plugins.env | tee -a /VERSION.txt \
+    && source /tmp/archivebox-enable-plugins.env \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups env HOME=/tmp/abxpkg-cache ABXPKG_NO_CACHE=True ABXPKG_TMP_CACHE_DIR=/tmp/abxpkg-cache archivebox install \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups env HOME=/tmp/abxpkg-cache archivebox version 2>&1 | tee -a /VERSION.txt \
+    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+
+RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
     && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
     ) | tee -a /VERSION.txt
 
-# Verify ArchiveBox is installed and write full version/dependency info.
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
-    && gosu "$ARCHIVEBOX_USER" archivebox install 2>&1 | tee -a /VERSION.txt \
-    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-####################################################
-
-# Expose ArchiveBox's main interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Nick Sweeting
+Copyright (c) 2026 Nick Sweeting
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -29,7 +29,7 @@ Without active preservation effort, everything on the internet eventually disapp
 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>
 
-> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  
+> ➡️ Get ArchiveBox with `pip install archivebox>=0.9.5rc1` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  
 
 *Once installed, you can interact with it through the: [Browser Extension](https://github.com/ArchiveBox/archivebox-browser-extension), [CLI](#usage), [self-hosted web interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python API](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [filesystem](#static-archive-exporting).*
 
@@ -92,7 +92,7 @@ docker run -it -v $PWD:/data archivebox/archivebox init
 <br/>
 <br/>
 # Option C: Or install it with your preferred pkg manager (see Quickstart below for apt, brew, and more)
-pip install archivebox
+pip install 'archivebox>=0.9.5rc1'
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
 archivebox install
@@ -106,7 +106,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
 <sub>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sub><br/>
-<sub>Set <code>LISTEN_HOST</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sub>
+<sub>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sub>
 </details>
 <br/>
 
@@ -295,18 +295,18 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
 <ol>
-<li>Download and install the <code>.deb</code> package from the <a href="https://github.com/ArchiveBox/ArchiveBox/releases">latest release</a>.
-<pre lang="bash"><code style="white-space: pre-line"># download the .deb for your architecture (amd64 or arm64)
-ARCH="$(dpkg --print-architecture)"
-VERSION="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'].lstrip('v'))")"
-curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${VERSION}_${ARCH}.deb" -o /tmp/archivebox.deb
-sudo apt install /tmp/archivebox.deb
-archivebox version                         # make sure all dependencies are installed
+<li>Add the third-party ArchiveBox apt repo and install <code>archivebox</code>.
+<pre lang="bash"><code style="white-space: pre-line">echo 'deb [trusted=yes] https://archivebox.github.io/debian-archivebox dev main' | sudo tee /etc/apt/sources.list.d/archivebox.list
+sudo apt update
+sudo apt install archivebox
+archivebox version                         # make sure the package is installed
 </code></pre>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --install
+archivebox init
+sudo archivebox install
+archivebox add 'https://example.com'
 </code></pre>
 <br/>
 </li>
@@ -476,6 +476,22 @@ For more discussion on managed and paid hosting options see here: <a href="https
 ArchiveBox commands can be run in a terminal [directly on your host](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), or via [Docker](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1)/[Docker Compose](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage).  
 <sup>(depending on how you chose to install it above)</sup>
 
+<!--
+```bash
+set -euo pipefail
+__archivebox_docs_home="$(mktemp -d)"
+export HOME="$__archivebox_docs_home"
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init
+archivebox version
+archivebox help
+test -f index.sqlite3
+test -d archive
+rm -rf "$__archivebox_docs_home"
+```
+-->
+<!--pytest.mark.skip(reason='Mixed host/Docker alternatives include interactive Docker commands')-->
 ```bash
 mkdir -p ~/archivebox/data   # create a new data dir anywhere
 cd ~/archivebox/data         # IMPORTANT: cd into the directory
@@ -512,10 +528,12 @@ archivebox persona create --import=chrome personal
 # make sure you have pip-installed ArchiveBox and it's available in your $PATH first  
 <br/>
 # archivebox [subcommand] [--help]
-archivebox init --install      # safe to run init multiple times (also how you update versions)
+mkdir -p ~/archivebox/data && cd ~/archivebox/data
+archivebox init
+sudo archivebox install
+archivebox add 'https://example.com'
 archivebox version           # get archivebox version info + check dependencies
 archivebox help              # get list of archivebox subcommands that can be run
-archivebox add --depth=1 'https://news.ycombinator.com'
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage">Usage: CLI Usage</a> wiki. ➡️</i>
 </details>
@@ -529,10 +547,11 @@ archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
 # docker compose run archivebox [subcommand] [--help]
-docker compose run archivebox init --install
+docker compose run archivebox init
+docker compose run archivebox install
 docker compose run archivebox version
 docker compose run archivebox help
-docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
+docker compose run archivebox add 'https://example.com'
 # to start webserver: docker compose up
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage">Usage: Docker Compose CLI</a> wiki. ➡️</i>
@@ -547,10 +566,11 @@ docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you create and cd into in a new empty directory first  
 <br/>
 # docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox init --install
+docker run -v $PWD:/data -it archivebox/archivebox init
+docker run -v $PWD:/data -it archivebox/archivebox install
 docker run -v $PWD:/data -it archivebox/archivebox version
 docker run -v $PWD:/data -it archivebox/archivebox help
-docker run -v $PWD:/data -it archivebox/archivebox add --depth=1 'https://news.ycombinator.com'
+docker run -v $PWD:/data -it archivebox/archivebox add 'https://example.com'
 # to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
@@ -588,7 +608,7 @@ docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 </code></pre>
 
 <sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
-<sup>Set <code>LISTEN_HOST</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sup>
+<sup>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sup>
 <br/><br/>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage">Usage: Web UI</a> wiki. ➡️</i>
 <br/><br/>
@@ -676,10 +696,36 @@ docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://exampl
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e1e5bd78-b0b6-45dc-914c-e1046fee4bc4" width="330px" align="right" style="float: right"/>
 
 
+<!--
+```bash
+set -euo pipefail
+__archivebox_docs_home="$(mktemp -d)"
+export HOME="$__archivebox_docs_home"
+mkdir -p ~/archivebox/data ~/Downloads
+cd ~/archivebox/data
+archivebox init
+cat > ~/Downloads/some_feed.xml <<'EOF'
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>ArchiveBox docs test feed</title>
+    <link>https://example.com/</link>
+    <description>ArchiveBox docs test feed</description>
+    <item>
+      <title>Feed item</title>
+      <link>https://example.com/from-feed</link>
+      <guid>https://example.com/from-feed</guid>
+    </item>
+  </channel>
+</rss>
+EOF
+```
+-->
+<!--pytest-codeblocks:cont-->
 ```bash
 # archivebox add --help
 archivebox add 'https://example.com/some/page'
-archivebox add --parser=generic_rss < ~/Downloads/some_feed.xml
+archivebox add --depth=1 --plugins=parse_rss_urls "file://$HOME/Downloads/some_feed.xml"
 archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
 echo 'http://example.com' | archivebox add
 echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add
@@ -689,6 +735,19 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
 # if using Docker Compose, add -T when piping stdin / stdout:
 # echo 'https://example.com' | docker compose run -T archivebox add
 ```
+<!--pytest-codeblocks:cont-->
+<!--
+```bash
+archivebox list --json > snapshots.json
+grep -q 'https://example.com/some/page' snapshots.json
+grep -q 'https://example.com/from-feed' snapshots.json
+grep -q 'https://news.ycombinator.com#2020-12-12' snapshots.json
+grep -q 'http://example.com' snapshots.json
+test -d archive
+test "$(find archive -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" -ge 4
+rm -rf "$__archivebox_docs_home"
+```
+-->
 
 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
 
@@ -1085,7 +1144,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
 </li>
 <li><strong>Try to keep the <code>data/index.sqlite3</code> file on local drive (not a network mount)</strong> or SSD for maximum performance, however the <code>data/archive/</code> folder can be on a network mount or slower HDD.</li>
-<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, you may need to set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid"><code>PUID</code> & <code>PGID</code></a> and <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304">disable <code>root_squash</code></a> on your fileshare server.
+<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, make sure the mounted data directory is writable by its intended owner and consider disabling <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304"><code>root_squash</code></a> on your fileshare server.
 </li>
 </ul>
 
@@ -1095,7 +1154,6 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid">Wiki: Configuration (<code>PUID</code> & <code>GUID</code>)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root">Wiki: Security Overview (Do Not Run as Root)</a></li>
 </ul>
 
@@ -1348,6 +1406,7 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc
 
 First make sure you have `uv` installed: https://docs.astral.sh/uv/getting-started/installation/
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 git clone https://github.com/ArchiveBox/monorepo
 cd monorepo
@@ -1369,6 +1428,7 @@ Repos included in monorepo setup:
 
 #### 2. Option A: Install the Python, JS, and system dependencies directly on your machine
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # Install ArchiveBox runtime dependencies
 mkdir -p data && cd data
@@ -1384,6 +1444,7 @@ archivebox server 0.0.0.0:8000
 
 #### 2. Option B: Build the docker container and use that for development instead
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # Optional: develop via docker by mounting the code dir into the container
 # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
@@ -1413,6 +1474,7 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # set up persistent DEBUG=True for all runs
 archivebox config --set DEBUG=True
@@ -1437,6 +1499,7 @@ https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-dj
 
 If you're looking for the latest `dev` Docker image, it's often available pre-built on Docker Hub, simply pull and use `archivebox/archivebox:dev`.
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 docker pull archivebox/archivebox:dev
 docker run archivebox/archivebox:dev version
@@ -1444,9 +1507,10 @@ docker run archivebox/archivebox:dev version
 ```
 
 ##### Build Branch from Source
-  
+
 You can also build and run any branch yourself from source, for example to build & use `dev` locally:
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # docker-compose.yml:
 services:
@@ -1471,6 +1535,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/lint.sh
 ./bin/test.sh
@@ -1484,6 +1549,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # generate the database migrations after changes to models.py
 cd archivebox/
@@ -1548,6 +1614,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/build.sh
 
@@ -1564,6 +1631,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/release.sh
 

archivebox/__init__.py

@@ -16,8 +16,6 @@ import sys
 from pathlib import Path
 from typing import Protocol, cast
 
-from abx_plugins import get_plugins_dir
-
 
 class _ReconfigurableStream(Protocol):
     def reconfigure(self, *, line_buffering: bool) -> object: ...
@@ -59,38 +57,59 @@ check_not_root()
 check_not_inside_source_dir()
 check_io_encoding()
 
-# Install monkey patches for third-party libraries
-from .misc.monkey_patches import *  # noqa
-
-# Plugin directories
-BUILTIN_PLUGINS_DIR = Path(get_plugins_dir()).resolve()
-USER_PLUGINS_DIR = (
-    Path(
-        os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or os.environ.get("USER_PLUGINS_DIR") or os.environ.get("DATA_DIR", os.getcwd()),
-    )
-    / "custom_plugins"
-)
-
-# These are kept for backwards compatibility with existing code
-# that checks for plugins. The new hook system uses discover_hooks()
-ALL_PLUGINS = {
-    "builtin": BUILTIN_PLUGINS_DIR,
-    "user": USER_PLUGINS_DIR,
-}
-LOADED_PLUGINS = ALL_PLUGINS
-
-# Setup basic config, constants, paths, and version
-from .config.constants import CONSTANTS  # noqa
-from .config.paths import PACKAGE_DIR, DATA_DIR  # noqa
 from .config.version import VERSION  # noqa
 
-# Set MACHINE_ID env var so hook scripts can use it
-os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)
 
 __version__ = VERSION
 __author__ = "ArchiveBox"
 __license__ = "MIT"
 
+
+def __getattr__(name: str):
+    if name == "CONSTANTS":
+        from .config.constants import CONSTANTS
+
+        os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)
+        return CONSTANTS
+    if name == "DATA_DIR":
+        from .config.paths import DATA_DIR
+
+        return DATA_DIR
+    if name == "VERSION":
+        return VERSION
+    if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
+        from abx_plugins import get_plugins_dir
+        from .config.constants import CONSTANTS
+
+        builtin_plugins_dir = Path(get_plugins_dir()).resolve()
+        user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
+        plugins = {
+            "builtin": builtin_plugins_dir,
+            "user": user_plugins_dir,
+        }
+        values = {
+            "BUILTIN_PLUGINS_DIR": builtin_plugins_dir,
+            "USER_PLUGINS_DIR": user_plugins_dir,
+            "ALL_PLUGINS": plugins,
+            "LOADED_PLUGINS": plugins,
+        }
+        return values[name]
+    raise AttributeError(name)
+
+
+__all__ = (
+    "ASCII_LOGO",
+    "ASCII_ICON",
+    "PACKAGE_DIR",
+    "DATA_DIR",
+    "CONSTANTS",
+    "VERSION",
+    "BUILTIN_PLUGINS_DIR",
+    "USER_PLUGINS_DIR",
+    "ALL_PLUGINS",
+    "LOADED_PLUGINS",
+)
+
 ASCII_ICON = """
 ██████████████████████████████████████████████████████████████████████████████████████████████████ 
 ██████████████████████████████████████████████████████████████████████████████████████████████████ 

archivebox/__main__.py

@@ -17,4 +17,4 @@ ASCII_LOGO_MINI = r"""
 """
 
 if __name__ == "__main__":
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main(args=sys.argv[1:])

archivebox/api/auth.py

@@ -7,7 +7,7 @@ from django.http import HttpRequest
 from django.contrib.auth import authenticate
 from django.contrib.auth.models import User
 
-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth
+from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader
 from ninja.errors import HttpError
 
 
@@ -51,6 +51,27 @@ def auth_using_token(token: str | None, request: HttpRequest | None = None) -> U
     return user
 
 
+def token_from_request(request: HttpRequest) -> str:
+    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
+    auth_header = request.headers.get("Authorization", "")
+    if not token and auth_header.lower().startswith("bearer "):
+        token = auth_header.split(None, 1)[1].strip()
+    return token
+
+
+def authenticated_user_from_request(request: HttpRequest) -> User | None:
+    user = request.user
+    if user.is_authenticated and user.is_active:
+        return user
+
+    token = token_from_request(request)
+    token_user = auth_using_token(token=token, request=request) if token else None
+    if token_user and token_user.is_active:
+        request.user = token_user
+        return token_user
+    return None
+
+
 def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
     """Given a username and password, check if they are valid and return the corresponding user"""
     user: User | None = None
@@ -106,33 +127,6 @@ class QueryParamTokenAuth(APIKeyQuery):
         return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
 
 
-class UsernameAndPasswordAuth(HttpBasicAuth):
-    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
-
-    def authenticate(self, request: HttpRequest, username: str, password: str) -> User | None:
-        return _require_superuser(
-            auth_using_password(username=username, password=password, request=request),
-            request,
-            self.__class__.__name__,
-        )
-
-
-class DjangoSessionAuth:
-    """Allow authenticating with existing Django session cookies (same-origin only)."""
-
-    def __call__(self, request: HttpRequest) -> User | None:
-        return self.authenticate(request)
-
-    def authenticate(self, request: HttpRequest, **kwargs) -> User | None:
-        user = getattr(request, "user", None)
-        if isinstance(user, User) and user.is_authenticated:
-            setattr(request, "_api_auth_method", self.__class__.__name__)
-            if not user.is_superuser:
-                raise HttpError(403, "Valid session but User does not have permission (make sure user.is_superuser=True)")
-            return user
-        return None
-
-
 ### Enabled Auth Methods
 
 API_AUTH_METHODS = [

archivebox/api/models.py

@@ -1,7 +1,7 @@
 __package__ = "archivebox.api"
 
 import secrets
-from archivebox.uuid_compat import uuid7
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 
 from django.conf import settings
 from django.db import models
@@ -17,7 +17,7 @@ def generate_secret_token() -> str:
 
 
 class APIToken(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
@@ -41,7 +41,7 @@ class APIToken(models.Model):
 
 
 class OutboundWebhook(WebhookBase):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)

archivebox/api/urls.py

@@ -7,8 +7,7 @@ from django.shortcuts import redirect
 from django.urls import path
 from django.views.generic.base import RedirectView
 
-from archivebox.core.host_utils import build_web_url
-
+from archivebox.core.routes_util import build_web_url
 from .v1_api import urls as v1_api_urls
 
 

archivebox/api/v1_api.py

@@ -68,14 +68,14 @@ class NinjaAPIWithIOCapture(NinjaAPI):
         # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
 
         # Add Auth Headers to response
-        api_token_attr = getattr(request, "_api_token", None)
+        api_token_attr = request.__dict__.get("_api_token")
         api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
         token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
 
-        response["X-ArchiveBox-Auth-Method"] = str(getattr(request, "_api_auth_method", "None"))
+        response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
         response["X-ArchiveBox-Auth-Expires"] = token_expiry
         response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
-        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if getattr(request.user, "pk", None) else "None"
+        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
         response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
 
         # import ipdb; ipdb.set_trace()

archivebox/api/v1_cli.py

@@ -8,10 +8,11 @@ from enum import Enum
 from django.http import HttpRequest
 
 from ninja import Router, Schema
+from ninja.errors import HttpError
 from pydantic import Field
 
 from archivebox.misc.util import ansi_to_html
-from archivebox.config.common import get_config
+from archivebox.core.models import SnapshotQuerySet
 
 
 # from .auth import API_AUTH_METHODS
@@ -23,6 +24,7 @@ router = Router(tags=["ArchiveBox CLI Sub-Commands"])
 # Schemas
 
 JSONType = list[Any] | dict[str, Any] | bool | int | str | None
+FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]
 
 
 class CLICommandResponseSchema(Schema):
@@ -34,26 +36,11 @@ class CLICommandResponseSchema(Schema):
     stderr: str
 
 
-class FilterTypeChoices(str, Enum):
-    exact = "exact"
-    substring = "substring"
-    regex = "regex"
-    domain = "domain"
-    tag = "tag"
-    timestamp = "timestamp"
-
-
-class StatusChoices(str, Enum):
-    indexed = "indexed"
-    archived = "archived"
-    unarchived = "unarchived"
-    present = "present"
-    valid = "valid"
-    invalid = "invalid"
-    duplicate = "duplicate"
-    orphaned = "orphaned"
-    corrupted = "corrupted"
-    unrecognized = "unrecognized"
+FilterTypeChoices = Enum(
+    "FilterTypeChoices",
+    {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
+    type=str,
+)
 
 
 class AddCommandSchema(Schema):
@@ -61,21 +48,39 @@ class AddCommandSchema(Schema):
     snapshot_ids: list[str] | None = None
     tag: str = ""
     depth: int = 0
+    max_urls: int = 0
+    crawl_max_size: int = 0
+    crawl_timeout: int = 0
+    snapshot_max_size: int = 0
     parser: str = "auto"
     plugins: str = ""
-    update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
+    only_new: bool | None = None
+    update: bool = False
     overwrite: bool = False
     index_only: bool = False
 
 
-class UpdateCommandSchema(Schema):
-    resume: str | None = None
+class SnapshotFilterCommandSchema(Schema):
     after: float | None = 0
-    before: float | None = 999999999999999
+    before: float | None = None
     filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = ["https://example.com"]
+    filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
+    status: str | None = None
+    url__icontains: str | None = None
+    url__istartswith: str | None = None
+    tag: str | None = None
+    crawl_id: str | None = None
+    limit: int | None = None
+    sort: str | None = None
+    search: str | None = None
+
+
+class UpdateCommandSchema(SnapshotFilterCommandSchema):
+    resume: str | None = None
     batch_size: int = 100
     continuous: bool = False
+    index_only: bool = False
+    migrate_only: bool = False
 
 
 class ScheduleCommandSchema(Schema):
@@ -88,48 +93,62 @@ class ScheduleCommandSchema(Schema):
     every: str | None = None
     tag: str = ""
     depth: int = 0
+    only_new: bool | None = None
+    update: bool = False
     overwrite: bool = False
-    update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
     clear: bool = False
 
 
-class ListCommandSchema(Schema):
-    filter_patterns: list[str] | None = ["https://example.com"]
-    filter_type: str = FilterTypeChoices.substring
-    status: StatusChoices = StatusChoices.indexed
-    after: float | None = 0
-    before: float | None = 999999999999999
-    sort: str = "bookmarked_at"
+class ListCommandSchema(SnapshotFilterCommandSchema):
     as_json: bool = True
     as_html: bool = False
     as_csv: str | None = "timestamp,url"
     with_headers: bool = False
 
 
-class RemoveCommandSchema(Schema):
-    delete: bool = True
-    after: float | None = 0
-    before: float | None = 999999999999999
+class RemoveCommandSchema(SnapshotFilterCommandSchema):
     filter_type: str = FilterTypeChoices.exact
-    filter_patterns: list[str] | None = ["https://example.com"]
+    timeout: float = 60.0
+
+
+def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
+    kwargs = args.dict()
+    kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
+    kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
+    return kwargs
 
 
 @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
 def cli_add(request: HttpRequest, args: AddCommandSchema):
     from archivebox.cli.archivebox_add import add
+    from archivebox.misc.util import validate_url
 
+    config_overrides: dict[str, object] = {}
+    if args.only_new is not None:
+        config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
+    submitted_urls: str | list[str] = args.urls
+    if len(args.urls) == 1:
+        try:
+            validate_url(args.urls[0])
+        except ValueError:
+            submitted_urls = args.urls[0]
     crawl, snapshots = add(
-        urls=args.urls,
+        urls=submitted_urls,
         snapshot_ids=args.snapshot_ids,
         tag=args.tag,
         depth=args.depth,
-        update=args.update,
+        max_urls=args.max_urls,
+        crawl_max_size=args.crawl_max_size,
+        crawl_timeout=args.crawl_timeout,
+        snapshot_max_size=args.snapshot_max_size,
         index_only=args.index_only,
-        overwrite=args.overwrite,
         plugins=args.plugins,
         parser=args.parser,
         bg=True,  # Always run in background for API calls
         created_by_id=request.user.pk,
+        config=config_overrides or None,
     )
 
     snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
@@ -139,8 +158,8 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         "snapshot_ids": snapshot_ids,
         "queued_urls": args.urls,
     }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
 
     return {
         "success": True,
@@ -154,23 +173,39 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
 
 @router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
 def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import update
+    from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
+    from archivebox.core.snapshot_status import normalize_snapshot_status
 
-    result = update(
-        filter_patterns=args.filter_patterns or [],
-        filter_type=args.filter_type or FilterTypeChoices.substring,
-        after=args.after,
-        before=args.before,
-        resume=args.resume,
-        batch_size=args.batch_size,
-        continuous=args.continuous,
+    try:
+        status = normalize_snapshot_status(args.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+
+    update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    update_kwargs["status"] = status
+    update_kwargs["stop_daemon_stack"] = False
+
+    is_filtered_update = any(
+        (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
     )
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    matched_snapshot_ids = []
+    if is_filtered_update:
+        matched_snapshot_ids = [
+            str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
+        ]
+
+    update(**update_kwargs)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
-        "result": result,
+        "result": {
+            "matched_count": len(matched_snapshot_ids),
+            "snapshot_ids": matched_snapshot_ids,
+        }
+        if is_filtered_update
+        else None,
         "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
         "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
     }
@@ -180,6 +215,11 @@ def cli_update(request: HttpRequest, args: UpdateCommandSchema):
 def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
     from archivebox.cli.archivebox_schedule import schedule
 
+    config_overrides: dict[str, object] = {}
+    if args.only_new is not None:
+        config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
     result = schedule(
         import_path=args.import_path,
         add=args.add,
@@ -191,12 +231,11 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
         every=args.every,
         tag=args.tag,
         depth=args.depth,
-        overwrite=args.overwrite,
-        update=args.update,
+        config=config_overrides or None,
     )
 
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
@@ -209,32 +248,38 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
 
 @router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
 def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_search import search
+    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
-    result = search(
-        filter_patterns=args.filter_patterns,
-        filter_type=args.filter_type,
-        status=args.status,
-        after=args.after,
-        before=args.before,
-        sort=args.sort,
-        csv=args.as_csv,
-        json=args.as_json,
-        html=args.as_html,
-        with_headers=args.with_headers,
-    )
+    search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    as_json = search_kwargs.pop("as_json")
+    as_html = search_kwargs.pop("as_html")
+    as_csv = search_kwargs.pop("as_csv")
+    with_headers = search_kwargs.pop("with_headers")
+    try:
+        snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
 
     result_format = "txt"
-    if args.as_json:
+    if as_json:
         result_format = "json"
-        result = json.loads(result)
-    elif args.as_html:
+        result = [
+            json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
+            for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
+        ]
+    elif as_html:
         result_format = "html"
-    elif args.as_csv:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
+    elif as_csv:
         result_format = "csv"
+        cols = [col.strip() for col in as_csv.split(",") if col.strip()]
+        rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
+        result = "\n".join((",".join(cols), *rows) if with_headers else rows)
+    else:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
 
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
@@ -248,38 +293,26 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
 @router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
 def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
     from archivebox.cli.archivebox_remove import remove
-    from archivebox.cli.archivebox_search import get_snapshots
     from archivebox.core.models import Snapshot
 
-    filter_patterns = args.filter_patterns or []
-    snapshots_to_remove = get_snapshots(
-        filter_patterns=filter_patterns,
-        filter_type=args.filter_type,
-        after=args.after,
-        before=args.before,
-    )
-    removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)]
+    remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
+    timeout_arg = remove_kwargs.pop("timeout")
+    timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
+    try:
+        snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
 
-    remove(
+    result = remove(
         yes=True,  # no way to interactively ask for confirmation via API, so we force yes
-        delete=args.delete,
         snapshots=snapshots_to_remove,
-        before=args.before,
-        after=args.after,
-        filter_type=args.filter_type,
-        filter_patterns=filter_patterns,
+        timeout=timeout,
     )
-
-    result = {
-        "removed_count": len(removed_snapshot_ids),
-        "removed_snapshot_ids": removed_snapshot_ids,
-        "remaining_snapshots": Snapshot.objects.count(),
-    }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
-        "success": True,
-        "errors": [],
+        "success": bool(result["success"]),
+        "errors": [str(result["error"])] if result["error"] else [],
         "result": result,
         "result_format": "json",
         "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",

archivebox/api/v1_core.py

@@ -4,6 +4,7 @@ import math
 import json
 import mimetypes
 import re
+from html import unescape
 from collections import defaultdict
 from pathlib import Path, PurePosixPath
 from uuid import UUID
@@ -11,8 +12,7 @@ from typing import Union, Any, Annotated
 from datetime import datetime, time
 
 from django.db import transaction
-from django.db.models import Model, Q, Sum
-from django.db.models.functions import Coalesce
+from django.db.models import Model, Q
 from django.http import HttpRequest, HttpResponse
 from django.http.multipartparser import MultiPartParser, MultiPartParserError
 from django.core.exceptions import ValidationError
@@ -29,11 +29,12 @@ from ninja.pagination import paginate, PaginationBase
 from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot, ArchiveResult, Tag
-from archivebox.api.auth import auth_using_token
+from archivebox.core.permissions import public_snapshots_queryset
+from archivebox.api.auth import authenticated_user_from_request
 from archivebox.config.common import get_config
-from archivebox.core.host_utils import build_web_url
-from archivebox.misc.util import validate_url_length
-from archivebox.core.tag_utils import (
+from archivebox.core.routes_util import build_web_url
+from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
+from archivebox.core.tag_util import (
     add_snapshot_counts,
     build_tag_cards,
     delete_tag as delete_tag_record,
@@ -49,7 +50,10 @@ from archivebox.core.tag_utils import (
     rename_tag as rename_tag_record,
 )
 from archivebox.crawls.models import Crawl
-from archivebox.api.v1_crawls import CrawlSchema
+from archivebox.api.v1_crawls import CrawlSchema, get_crawl_by_ref
+from archivebox.search.config import get_search_mode, get_search_mode_backend
+from archivebox.search.query import apply_snapshot_search
+from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status
 
 
 router = Router(tags=["Core Models"])
@@ -77,7 +81,7 @@ class CustomPagination(PaginationBase):
     def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
         limit = min(pagination.limit, 500)
         offset = pagination.offset or (pagination.page * limit)
-        total = queryset.count()
+        total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count()
         total_pages = math.ceil(total / limit)
         current_page = math.ceil(offset / (limit + 1))
         items = queryset[offset : offset + limit]
@@ -210,7 +214,10 @@ class ArchiveResultFilterSchema(FilterSchema):
 @paginate(CustomPagination)
 def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
     """List all ArchiveResult entries matching these filters."""
-    return filters.filter(ArchiveResult.objects.all()).distinct()
+    queryset = filters.filter(ArchiveResult.objects.all())
+    if filters.search or filters.snapshot_tag:
+        return queryset.distinct()
+    return queryset
 
 
 def _uuid_ref_query(field_name: str, ref: str) -> Q:
@@ -271,7 +278,7 @@ def _parse_archiveresult_output_json(output_json: str | None) -> dict[str, Any]
 
 
 def _get_archiveresult_upload_data(request: HttpRequest):
-    cached = getattr(request, "_archiveresult_upload_data", None)
+    cached = request.__dict__.get("_archiveresult_upload_data")
     if cached is not None:
         return cached
 
@@ -359,23 +366,26 @@ def _get_snapshot_by_ref(snapshot_id: str):
         return queryset.get(_uuid_ref_query("id", snapshot_id))
 
 
-def _touch_archiveresult_snapshot(snapshot: Snapshot) -> None:
-    now = timezone.now()
-    snapshot_updates = ["modified_at"]
-    if snapshot.downloaded_at is None:
-        snapshot.downloaded_at = now
-        snapshot_updates.append("downloaded_at")
-    if snapshot.status != Snapshot.StatusChoices.SEALED:
-        snapshot.status = Snapshot.StatusChoices.SEALED
-        snapshot.retry_at = None
-        snapshot_updates.extend(["status", "retry_at"])
-    snapshot.save(update_fields=snapshot_updates)
+def _queue_archiveresult_snapshot_maintenance(snapshot: Snapshot) -> None:
+    """
+    Mark an uploaded ArchiveResult's Snapshot as dirty without finalizing it.
 
-    try:
-        snapshot.ensure_crawl_symlink(snapshot_dir=snapshot.output_dir)
-        snapshot.write_index_jsonl(output_dir=snapshot.output_dir)
-    except Exception:
-        pass
+    Upload API handlers are allowed to persist files and ArchiveResult rows, but
+    Snapshot save() side effects, sealing, symlink creation, and index/details
+    rewrites belong to the runner. retry_at is the scheduler signal the runner
+    already watches, so only bump rows that are final or otherwise invisible.
+    """
+    # ArchiveResult.save() updates parent snapshot health/mtime before this
+    # helper runs. Re-read the scheduler columns so the short CAS update below
+    # does not lose to our own earlier ArchiveResult write.
+    snapshot = Snapshot.objects.only("id", "status", "retry_at", "downloaded_at", "modified_at").get(id=snapshot.id)
+    now = timezone.now()
+    updates = {"modified_at": now}
+    if snapshot.downloaded_at is None:
+        updates["downloaded_at"] = now
+    if snapshot.status == Snapshot.StatusChoices.SEALED or snapshot.retry_at is None:
+        updates["retry_at"] = now
+    snapshot.safe_update(updates, refresh=False)
 
 
 def _merge_archiveresult_output_file_maps(results: list[ArchiveResult]) -> dict[str, dict[str, Any]]:
@@ -463,12 +473,7 @@ def _write_archiveresult_files(
             )
 
         guessed_mime = mimetypes.guess_type(relative_output_path)[0]
-        output_mime_type = (
-            (mime_types[0] if mime_types else "")
-            or getattr(uploaded_file, "content_type", None)
-            or guessed_mime
-            or "application/octet-stream"
-        )
+        output_mime_type = (mime_types[0] if mime_types else "") or uploaded_file.content_type or guessed_mime or "application/octet-stream"
         output_files[relative_output_path] = {
             "extension": PurePosixPath(relative_output_path).suffix.lower().lstrip("."),
             "mimetype": output_mime_type,
@@ -496,7 +501,7 @@ def _write_archiveresult_files(
         guessed_mime = mimetypes.guess_type(saved_output_path)[0]
         output_mime_type = (
             (mime_types[index] if index < len(mime_types) else "")
-            or getattr(uploaded_file, "content_type", None)
+            or uploaded_file.content_type
             or guessed_mime
             or "application/octet-stream"
         )
@@ -530,13 +535,11 @@ def create_archiveresult(
     parsed_output_json = _parse_archiveresult_output_json(output_json)
     hook = hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME
     matching_results = list(
-        ArchiveResult.objects.select_for_update()
-        .filter(
+        ArchiveResult.objects.filter(
             snapshot=snapshot,
             plugin=plugin_name,
             hook_name=hook,
-        )
-        .order_by("created_at", "id"),
+        ).order_by("created_at", "id"),
     )
     existing_result = matching_results[0] if matching_results else None
     existing_output_files = _merge_archiveresult_output_file_maps(matching_results)
@@ -547,29 +550,42 @@ def create_archiveresult(
         existing_output_files=existing_output_files,
         allow_empty=True,
     )
-    output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
-    output_file_paths = list(output_files.keys())
     now = timezone.now()
-    if (
-        existing_result
-        and normalized_status == ArchiveResult.StatusChoices.STARTED
-        and existing_result.status != ArchiveResult.StatusChoices.STARTED
-    ):
-        normalized_status = existing_result.status
 
     with transaction.atomic():
-        if existing_result:
+        matching_results = list(
+            ArchiveResult.objects.filter(
+                snapshot=snapshot,
+                plugin=plugin_name,
+                hook_name=hook,
+            ).order_by("created_at", "id"),
+        )
+        if matching_results:
+            existing_result = matching_results[0]
+            output_files = {
+                **_merge_archiveresult_output_file_maps(matching_results),
+                **output_files,
+            }
             duplicate_ids = [result.id for result in matching_results[1:]]
             if duplicate_ids:
                 ArchiveResult.objects.filter(id__in=duplicate_ids).delete()
             result = existing_result
         else:
+            existing_result = None
             result = ArchiveResult(
                 snapshot=snapshot,
                 plugin=plugin_name,
                 hook_name=hook,
             )
 
+        if (
+            existing_result
+            and normalized_status == ArchiveResult.StatusChoices.STARTED
+            and existing_result.status != ArchiveResult.StatusChoices.STARTED
+        ):
+            normalized_status = existing_result.status
+        output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
+        output_file_paths = list(output_files.keys())
         result.status = normalized_status
         result.output_str = output_str or (output_file_paths[0] if output_file_paths else "")
         result.output_json = parsed_output_json
@@ -581,7 +597,7 @@ def create_archiveresult(
         result.save()
 
     if result.status != ArchiveResult.StatusChoices.STARTED:
-        _touch_archiveresult_snapshot(snapshot)
+        _queue_archiveresult_snapshot_maintenance(snapshot)
     return result
 
 
@@ -622,13 +638,16 @@ def patch_archiveresult(
             normalized_status = latest_result.status
         result.status = normalized_status
         update_fields.append("status")
+    elif latest_result.status == ArchiveResult.StatusChoices.QUEUED and ArchiveResult.output_files_upload_complete(output_files):
+        result.status = ArchiveResult.StatusChoices.SUCCEEDED
+        update_fields.append("status")
     if output_json:
         result.output_json = _parse_archiveresult_output_json(output_json)
         update_fields.append("output_json")
 
     result.save(update_fields=update_fields)
     if result.status != ArchiveResult.StatusChoices.STARTED:
-        _touch_archiveresult_snapshot(result.snapshot)
+        _queue_archiveresult_snapshot_maintenance(result.snapshot)
 
     return result
 
@@ -671,7 +690,7 @@ class SnapshotSchema(Schema):
 
     @staticmethod
     def resolve_archive_size(obj):
-        return int(getattr(obj, "output_size_sum", obj.archive_size) or 0)
+        return int(obj.archive_size or 0)
 
     @staticmethod
     def resolve_output_size(obj):
@@ -683,12 +702,13 @@ class SnapshotSchema(Schema):
 
     @staticmethod
     def resolve_archiveresults(obj, context):
-        if bool(getattr(context["request"], "with_archiveresults", False)):
+        if bool(context["request"].__dict__.get("with_archiveresults", False)):
             return obj.archiveresult_set.all().distinct()
         return ArchiveResult.objects.none()
 
 
 class SnapshotUpdateSchema(Schema):
+    action: str | None = None
     status: str | None = None
     retry_at: datetime | None = None
     tags: list[str] | None = None
@@ -766,10 +786,10 @@ def _filter_snapshots_for_rss(
         )
         .filter(bookmarked_at__lte=before_dt)
     )
-
     crawl_id = crawl_id.strip()
     if crawl_id:
-        queryset = queryset.filter(crawl__id__icontains=crawl_id)
+        matching_crawl_pks = list(filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).values_list("pk", flat=True)[:100])
+        queryset = queryset.filter(crawl_id__in=matching_crawl_pks)
 
     created_by = created_by.strip()
     if created_by:
@@ -813,7 +833,7 @@ def _snapshots_rss_response(
         crawl_user = snapshot.crawl.created_by if snapshot.crawl_id else None
         description = f"Original URL: {snapshot.url}\nArchived snapshot: {archived_url}"
         feed.add_item(
-            title=snapshot.title or snapshot.url,
+            title=unescape(snapshot.title or snapshot.url),
             link=archived_url or web_base_url,
             description=description,
             unique_id=str(snapshot.id),
@@ -828,7 +848,7 @@ def _snapshots_rss_response(
 
 
 class SnapshotFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup(["id__icontains", "timestamp__startswith"])] = None
+    id: Annotated[str | None, FilterLookup(["id__istartswith", "id__iendswith", "timestamp__startswith"])] = None
     created_by_id: Annotated[str | None, FilterLookup("crawl__created_by_id")] = None
     created_by_username: Annotated[str | None, FilterLookup("crawl__created_by__username__icontains")] = None
     created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
@@ -837,10 +857,9 @@ class SnapshotFilterSchema(FilterSchema):
     modified_at: Annotated[datetime | None, FilterLookup("modified_at")] = None
     modified_at__gte: Annotated[datetime | None, FilterLookup("modified_at__gte")] = None
     modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
-    search: Annotated[
-        str | None,
-        FilterLookup(["url__icontains", "title__icontains", "tags__name__icontains", "id__icontains", "timestamp__startswith"]),
-    ] = None
+    search: str | None = None
+    search_mode: str | None = None
+    status: str | None = None
     url: Annotated[str | None, FilterLookup("url")] = None
     tag: Annotated[str | None, FilterLookup("tags__name")] = None
     title: Annotated[str | None, FilterLookup("title__icontains")] = None
@@ -848,14 +867,44 @@ class SnapshotFilterSchema(FilterSchema):
     bookmarked_at__gte: Annotated[datetime | None, FilterLookup("bookmarked_at__gte")] = None
     bookmarked_at__lt: Annotated[datetime | None, FilterLookup("bookmarked_at__lt")] = None
 
+    def filter_search(self, value: str | None) -> Q:
+        return Q()
+
+    def filter_search_mode(self, value: str | None) -> Q:
+        return Q()
+
+    def filter_status(self, value: str | None) -> Q:
+        return Q()
+
 
 @router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
 @paginate(CustomPagination)
 def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
     """List all Snapshot entries matching these filters."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0))
-    return filters.filter(queryset).distinct()
+    try:
+        queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+    queryset = filters.filter(queryset).distinct()
+    query = (filters.search or "").strip()
+    if not query:
+        return queryset
+
+    runtime_config = request.archivebox_config
+    search_mode = get_search_mode(filters.search_mode, config=runtime_config)
+    try:
+        return apply_snapshot_search(
+            queryset,
+            query,
+            search_mode=search_mode,
+            config=runtime_config,
+            include_id_matches=True,
+        )
+    except Exception:
+        if get_search_mode_backend(search_mode, config=runtime_config):
+            return queryset.none()
+        return apply_snapshot_search(queryset, query, search_mode="meta", config=runtime_config, include_id_matches=True)
 
 
 @router.get("/snapshots.rss", url_name="get_snapshots_rss")
@@ -880,18 +929,16 @@ def get_snapshots_rss(
 def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
     """Get a specific Snapshot by id."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0))
-    try:
-        return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        return queryset.get(_uuid_ref_query("id", snapshot_id))
+    return _get_snapshot_by_ref(snapshot_id)
 
 
 @router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
 def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     tags = normalize_tag_list(data.tags)
-    if data.status is not None and data.status not in Snapshot.StatusChoices.values:
-        raise HttpError(400, f"Invalid status: {data.status}")
+    try:
+        status = normalize_snapshot_status(data.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
     if not data.url.strip():
         raise HttpError(400, "URL is required")
     try:
@@ -902,7 +949,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
         raise HttpError(400, "depth must be between 0 and 4")
 
     if data.crawl_id:
-        crawl = Crawl.objects.get(id__icontains=data.crawl_id)
+        crawl = get_crawl_by_ref(data.crawl_id)
         crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
         tags = tags or crawl_tags
     else:
@@ -919,7 +966,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
         "depth": data.depth,
         "title": data.title,
         "timestamp": str(timezone.now().timestamp()),
-        "status": data.status or Snapshot.StatusChoices.QUEUED,
+        "status": status or Snapshot.StatusChoices.QUEUED,
         "retry_at": timezone.now(),
     }
     snapshot, _ = Snapshot.objects.get_or_create(
@@ -932,10 +979,8 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     if data.title is not None and snapshot.title != data.title:
         snapshot.title = data.title
         update_fields.append("title")
-    if data.status is not None and snapshot.status != data.status:
-        if data.status not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {data.status}")
-        snapshot.status = data.status
+    if status is not None and snapshot.status != status:
+        snapshot.status = status
         update_fields.append("status")
     if update_fields:
         update_fields.append("modified_at")
@@ -956,19 +1001,33 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
 @router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
 def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
     """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
-    try:
-        snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        snapshot = Snapshot.objects.get(Q(id__icontains=snapshot_id))
+    snapshot = _get_snapshot_by_ref(snapshot_id)
 
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
+    action = payload.pop("action", None)
     tags = payload.pop("tags", None)
 
+    if action:
+        if action == "pause":
+            snapshot.pause()
+            setattr(request, "with_archiveresults", False)
+            return snapshot
+        if action in ("resume", "unpause"):
+            snapshot.resume()
+            setattr(request, "with_archiveresults", False)
+            return snapshot
+        if action == "cancel":
+            snapshot.cancel()
+            setattr(request, "with_archiveresults", False)
+            return snapshot
+        raise HttpError(400, f"Invalid action: {action}")
+
     if "status" in payload:
-        if payload["status"] not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        snapshot.status = payload["status"]
+        try:
+            snapshot.status = normalize_snapshot_status(payload["status"])
+        except ValueError as err:
+            raise HttpError(400, str(err)) from err
         if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
             snapshot.retry_at = None
         update_fields.append("status")
@@ -980,7 +1039,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
     if tags is not None:
         snapshot.save_tags(normalize_tag_list(tags))
 
-    snapshot.save(update_fields=update_fields)
+    if payload.get("status") == Snapshot.StatusChoices.SEALED:
+        snapshot.cancel()
+    else:
+        snapshot.save(update_fields=update_fields)
     setattr(request, "with_archiveresults", False)
     return snapshot
 
@@ -1021,7 +1083,7 @@ class TagSchema(Schema):
     def resolve_created_by_username(obj):
         user_model = get_user_model()
         user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
+        username = user.username
         return username if isinstance(username, str) else str(user)
 
     @staticmethod
@@ -1030,7 +1092,7 @@ class TagSchema(Schema):
 
     @staticmethod
     def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
+        if bool(context["request"].__dict__.get("with_snapshots", False)):
             return obj.snapshot_set.all().distinct()
         return Snapshot.objects.none()
 
@@ -1164,6 +1226,40 @@ class TagSnapshotResponseSchema(Schema):
     tag_name: str
 
 
+def _get_snapshot_for_tag_edit(snapshot_ref: str) -> Snapshot:
+    snapshot_ref = str(snapshot_ref or "").strip().lower()
+    if not snapshot_ref:
+        raise HttpError(400, "Snapshot id is required")
+
+    snapshot_qs = Snapshot.objects.only("id")
+    is_full_uuid = len(snapshot_ref.replace("-", "")) == 32 and all(char in "0123456789abcdef-" for char in snapshot_ref)
+    if is_full_uuid:
+        try:
+            return snapshot_qs.get(pk=snapshot_ref.replace("-", ""))
+        except (Snapshot.DoesNotExist, ValueError):
+            pass
+
+    if len(snapshot_ref) >= 14:
+        try:
+            return snapshot_qs.get(timestamp=snapshot_ref)
+        except Snapshot.DoesNotExist:
+            pass
+        except Snapshot.MultipleObjectsReturned:
+            snapshot = snapshot_qs.filter(timestamp=snapshot_ref).first()
+            if snapshot is not None:
+                return snapshot
+
+    try:
+        return snapshot_qs.get(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref))
+    except Snapshot.DoesNotExist:
+        raise HttpError(404, "Snapshot not found") from None
+    except Snapshot.MultipleObjectsReturned:
+        snapshot = snapshot_qs.filter(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref)).first()
+        if snapshot is None:
+            raise HttpError(404, "Snapshot not found")
+        return snapshot
+
+
 @router.get("/tags/search/", response=TagSearchResponseSchema, url_name="search_tags")
 def search_tags(
     request: HttpRequest,
@@ -1182,6 +1278,7 @@ def search_tags(
         "tags": build_tag_cards(
             query=q,
             request=request,
+            preview_limit=0,
             sort=normalized_sort,
             created_by=normalized_created_by,
             year=normalized_year,
@@ -1195,23 +1292,11 @@ def search_tags(
 
 
 def _public_tag_listing_enabled() -> bool:
-    config = get_config()
-    if config.PUBLIC_SNAPSHOTS_LIST is not None:
-        return config.PUBLIC_SNAPSHOTS_LIST
-    return config.PUBLIC_INDEX
+    return get_config().PUBLIC_INDEX
 
 
 def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
-    user = getattr(request, "user", None)
-    if getattr(user, "is_authenticated", False):
-        return True
-
-    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-    auth_header = request.headers.get("Authorization", "")
-    if not token and auth_header.lower().startswith("bearer "):
-        token = auth_header.split(None, 1)[1].strip()
-
-    if token and auth_using_token(token=token, request=request):
+    if authenticated_user_from_request(request):
         return True
 
     return _public_tag_listing_enabled()
@@ -1223,11 +1308,16 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
     if not _request_has_tag_autocomplete_access(request):
         raise HttpError(401, "Authentication required")
 
-    tags = list(get_matching_tags(q, with_snapshot_counts=False)[: 50 if not q else 20])
-    add_snapshot_counts(tags)
+    public_only = not request.user.is_authenticated and not request.__dict__.get("_api_token")
+    queryset = get_matching_tags(q)
+    public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
+    if public_only:
+        queryset = queryset.filter(snapshot_set__id__in=public_snapshots.values("id")).distinct()
+    tags = list(queryset[: 50 if not q else 20])
+    add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)
 
     return {
-        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": getattr(tag, "num_snapshots", 0)} for tag in tags],
+        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": tag.__dict__.get("num_snapshots", 0)} for tag in tags],
     }
 
 
@@ -1308,19 +1398,7 @@ def tag_snapshots_export(request: HttpRequest, tag_id: int):
 @router.post("/tags/add-to-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_add_to_snapshot")
 def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
     """Add a tag to a snapshot. Creates the tag if it doesn't exist."""
-    # Get the snapshot
-    try:
-        snapshot = Snapshot.objects.get(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        )
-    except Snapshot.DoesNotExist:
-        raise HttpError(404, "Snapshot not found")
-    except Snapshot.MultipleObjectsReturned:
-        snapshot = Snapshot.objects.filter(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        ).first()
-        if snapshot is None:
-            raise HttpError(404, "Snapshot not found")
+    snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
 
     # Get or create the tag
     if data.tag_name:
@@ -1352,19 +1430,7 @@ def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
 @router.post("/tags/remove-from-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_remove_from_snapshot")
 def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
     """Remove a tag from a snapshot."""
-    # Get the snapshot
-    try:
-        snapshot = Snapshot.objects.get(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        )
-    except Snapshot.DoesNotExist:
-        raise HttpError(404, "Snapshot not found")
-    except Snapshot.MultipleObjectsReturned:
-        snapshot = Snapshot.objects.filter(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        ).first()
-        if snapshot is None:
-            raise HttpError(404, "Snapshot not found")
+    snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
 
     # Get the tag
     if data.tag_id:

archivebox/api/v1_crawls.py

@@ -1,8 +1,10 @@
 __package__ = "archivebox.api"
 
+import json
+from pathlib import Path
 from uuid import UUID
 from datetime import datetime
-from django.http import HttpRequest
+from django.http import FileResponse, HttpRequest
 from django.shortcuts import redirect
 from django.utils import timezone
 
@@ -13,10 +15,17 @@ from ninja import Router, Schema
 from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot
-from archivebox.crawls.models import Crawl
+from archivebox.core.permissions import (
+    PERMISSIONS_PUBLIC,
+    PERMISSIONS_UNLISTED,
+    is_admin_user,
+    normalize_permissions,
+)
 from archivebox.config.common import get_config
+from archivebox.crawls.models import Crawl
+from archivebox.misc.util import filter_queryset_by_uuid_substring
 
-from .auth import API_AUTH_METHODS
+from .auth import API_AUTH_METHODS, authenticated_user_from_request
 
 router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
 
@@ -33,22 +42,15 @@ class CrawlSchema(Schema):
 
     status: str
     retry_at: datetime | None
+    is_paused: bool
 
     urls: str
     max_depth: int
-    max_urls: int
-    crawl_max_size: int
-    snapshot_max_size: int
-    crawl_max_concurrent_snapshots: int
     tags_str: str
     config: dict
 
     # snapshots: List[SnapshotSchema]
 
-    @staticmethod
-    def resolve_crawl_max_concurrent_snapshots(obj):
-        return int((obj.config or {}).get("CRAWL_MAX_CONCURRENT_SNAPSHOTS") or get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS)
-
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
@@ -57,17 +59,26 @@ class CrawlSchema(Schema):
     def resolve_created_by_username(obj):
         user_model = get_user_model()
         user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
+        username = user.username
         return username if isinstance(username, str) else str(user)
 
+    @staticmethod
+    def resolve_config(obj):
+        # Redact credential values so REST responses can never leak the raw
+        # token/secret/api-key that the operator stored in Crawl.config.
+        from archivebox.config.common import redact_sensitive_config
+
+        return redact_sensitive_config(obj.config)
+
     @staticmethod
     def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
+        if bool(context["request"].__dict__.get("with_snapshots", False)):
             return obj.snapshot_set.all().distinct()
         return Snapshot.objects.none()
 
 
 class CrawlUpdateSchema(Schema):
+    action: str | None = None
     status: str | None = None
     retry_at: datetime | None = None
     tags: list[str] | None = None
@@ -77,10 +88,6 @@ class CrawlUpdateSchema(Schema):
 class CrawlCreateSchema(Schema):
     urls: list[str]
     max_depth: int = 0
-    max_urls: int = 0
-    crawl_max_size: int = 0
-    snapshot_max_size: int = 0
-    crawl_max_concurrent_snapshots: int | None = None
     tags: list[str] | None = None
     tags_str: str = ""
     label: str = ""
@@ -113,27 +120,13 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
         raise HttpError(400, "At least one URL is required")
     if data.max_depth not in (0, 1, 2, 3, 4):
         raise HttpError(400, "max_depth must be between 0 and 4")
-    if data.max_urls < 0:
-        raise HttpError(400, "max_urls must be >= 0")
-    if data.crawl_max_size < 0:
-        raise HttpError(400, "crawl_max_size must be >= 0")
-    if data.snapshot_max_size < 0:
-        raise HttpError(400, "snapshot_max_size must be >= 0")
-    crawl_max_concurrent_snapshots = data.crawl_max_concurrent_snapshots
-    if crawl_max_concurrent_snapshots is None:
-        crawl_max_concurrent_snapshots = get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS
-    if crawl_max_concurrent_snapshots < 1:
-        raise HttpError(400, "crawl_max_concurrent_snapshots must be >= 1")
 
     tags = normalize_tag_list(data.tags, data.tags_str)
     config = dict(data.config or {})
-    config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
+    config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
     crawl = Crawl.objects.create(
-        urls="\n".join(urls),
+        urls="\n".join(json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in urls),
         max_depth=data.max_depth,
-        max_urls=data.max_urls,
-        crawl_max_size=data.crawl_max_size,
-        snapshot_max_size=data.snapshot_max_size,
         tags_str=",".join(tags),
         label=data.label,
         notes=data.notes,
@@ -142,16 +135,19 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
         retry_at=timezone.now(),
         created_by=request.user if isinstance(request.user, User) else None,
     )
-    crawl.create_snapshots_from_urls()
     return crawl
 
 
+def get_crawl_by_ref(crawl_id: str):
+    return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+
+
 @router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
 def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
     """Get a specific Crawl by id."""
     setattr(request, "with_snapshots", with_snapshots)
     setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl = get_crawl_by_ref(crawl_id)
 
     if crawl and as_rss:
         query = request.GET.copy()
@@ -162,13 +158,72 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn
     return crawl
 
 
+def crawl_file(request: HttpRequest, crawl_id: str, path: str):
+    # Try to resolve the crawl first; if it doesn't exist, return 404.
+    try:
+        crawl = get_crawl_by_ref(crawl_id)
+    except Crawl.DoesNotExist:
+        raise HttpError(404, "Crawl not found")
+
+    user = authenticated_user_from_request(request)
+
+    # Gate access using the same model as SnapshotView/can_view_snapshot:
+    # admins always pass; owners can see their own crawls; otherwise the crawl
+    # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
+    if not is_admin_user(request):
+        permissions = normalize_permissions(crawl.permissions)
+        is_owner = bool(user and crawl.created_by_id == user.id)
+        if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
+            raise HttpError(404, "Crawl not found")
+
+    crawl_root = Path(crawl.output_dir).resolve()
+    file_path = (crawl_root / path).resolve()
+    if not file_path.is_file() or crawl_root not in file_path.parents:
+        raise HttpError(404, "Crawl file not found")
+
+    response = FileResponse(file_path.open("rb"))
+    response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
+    response["Pragma"] = "no-cache"
+    response["Expires"] = "0"
+    response["X-Content-Type-Options"] = "nosniff"
+    return response
+
+
+@router.get("/crawl/{crawl_id}/files/{filename}", auth=None, url_name="crawl_file_root")
+def crawl_file_root(request: HttpRequest, crawl_id: str, filename: str):
+    return crawl_file(request, crawl_id, filename)
+
+
+@router.get("/crawl/{crawl_id}/files/{folder}/{filename}", auth=None, url_name="crawl_file_nested_1")
+def crawl_file_nested_1(request: HttpRequest, crawl_id: str, folder: str, filename: str):
+    return crawl_file(request, crawl_id, f"{folder}/{filename}")
+
+
+@router.get("/crawl/{crawl_id}/files/{folder}/{subfolder}/{filename}", auth=None, url_name="crawl_file_nested_2")
+def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfolder: str, filename: str):
+    return crawl_file(request, crawl_id, f"{folder}/{subfolder}/{filename}")
+
+
 @router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
 def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
     """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl = get_crawl_by_ref(crawl_id)
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
 
+    action = payload.pop("action", None)
+    if action:
+        if action == "pause":
+            crawl.pause()
+            return crawl
+        if action in ("resume", "unpause"):
+            crawl.resume()
+            return crawl
+        if action == "cancel":
+            crawl.cancel()
+            return crawl
+        raise HttpError(400, f"Invalid action: {action}")
+
     tags = payload.pop("tags", None)
     tags_str = payload.pop("tags_str", None)
     if tags is not None or tags_str is not None:
@@ -178,37 +233,23 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
     if "status" in payload:
         if payload["status"] not in Crawl.StatusChoices.values:
             raise HttpError(400, f"Invalid status: {payload['status']}")
+        if payload["status"] == Crawl.StatusChoices.SEALED:
+            crawl.cancel()
+            return crawl
         crawl.status = payload["status"]
-        if crawl.status == Crawl.StatusChoices.SEALED and "retry_at" not in payload:
-            crawl.retry_at = None
         update_fields.append("status")
 
     if "retry_at" in payload:
         crawl.retry_at = payload["retry_at"]
         update_fields.append("retry_at")
 
-    if payload.get("status") == Crawl.StatusChoices.SEALED:
-        cancelled_at = timezone.now()
-        crawl.retry_at = None
-        if "retry_at" not in update_fields:
-            update_fields.append("retry_at")
-        crawl.save(update_fields=update_fields)
-        Snapshot.objects.filter(
-            crawl=crawl,
-            status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
-        ).update(
-            status=Snapshot.StatusChoices.SEALED,
-            retry_at=None,
-            modified_at=cancelled_at,
-        )
-    else:
-        crawl.save(update_fields=update_fields)
+    crawl.save(update_fields=update_fields)
     return crawl
 
 
 @router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
 def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl = get_crawl_by_ref(crawl_id)
     crawl_id_str = str(crawl.id)
     snapshot_count = crawl.snapshot_set.count()
     deleted_count, _ = crawl.delete()

archivebox/api/v1_personas.py

@@ -8,8 +8,10 @@ from uuid import UUID
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.pagination import paginate
 from pydantic import Field
 
+from archivebox.api.v1_core import CustomPagination
 from archivebox.personas.importers import validate_persona_name
 from archivebox.personas.models import Persona
 
@@ -51,6 +53,14 @@ class PersonaSchema(Schema):
     def resolve_created_by_username(obj) -> str:
         return obj.created_by.username
 
+    @staticmethod
+    def resolve_config(obj):
+        # Redact credential values so REST responses don't leak the raw
+        # token/secret/api-key the operator stored in Persona.config.
+        from archivebox.config.common import redact_sensitive_config
+
+        return redact_sensitive_config(obj.config)
+
 
 class PersonaSyncResponseSchema(Schema):
     success: bool
@@ -105,6 +115,7 @@ def find_persona(extension_persona_id: str, name: str) -> Persona | None:
 
 
 @router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
+@paginate(CustomPagination)
 def get_personas(request: HttpRequest):
     """List personas available on this ArchiveBox server."""
     return Persona.objects.all().order_by("name")
@@ -128,7 +139,7 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     created = persona is None
     if persona is None:
         persona = Persona(name=name)
-        if getattr(request.user, "is_authenticated", False):
+        if request.user.is_authenticated:
             persona.created_by = request.user
 
     persona.config = {

archivebox/base_models/admin.py

@@ -3,18 +3,50 @@
 __package__ = "archivebox.base_models"
 
 import json
+import uuid
 from collections.abc import Mapping
-from typing import NotRequired, TypedDict
+from typing import NotRequired, TypedDict, cast
 
 from django import forms
 from django.contrib import admin
 from django.db import models
 from django.forms.renderers import BaseRenderer
 from django.http import HttpRequest, QueryDict
+from django.urls import path, register_converter
 from django.utils.safestring import SafeString, mark_safe
 from django_object_actions import DjangoObjectActions
 
 
+class HexUUIDConverter:
+    """URL path converter that canonicalizes UUIDs to their 32-char hex form.
+
+    Accepts both the hyphenated (``aaaaaaaa-bbbb-...``) and bare-hex
+    (``aaaaaaaabbbb...``) UUID strings on the way in (Django's UUIDField
+    parses either), but ``to_url`` always emits the bare-hex form. This is
+    what makes ``reverse("admin:app_model_change", args=[obj.pk])`` produce
+    ``/admin/app/model/06a1a8facb0d.../change/`` instead of the default
+    hyphenated rendering — admin links throughout the app reverse through
+    this converter once ``BaseModelAdmin.get_urls`` swaps in
+    ``<hexuuid:object_id>`` below.
+    """
+
+    regex = r"[0-9a-fA-F]{32}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
+
+    def to_python(self, value: str) -> str:
+        # Strip hyphens but stay as a string — Django admin treats the
+        # captured object_id as a string and calls ``model._meta.pk.to_python``
+        # itself, so we don't want to short-circuit that.
+        return value.replace("-", "")
+
+    def to_url(self, value) -> str:
+        if isinstance(value, uuid.UUID):
+            return value.hex
+        return str(value).replace("-", "")
+
+
+register_converter(HexUUIDConverter, "hexuuid")
+
+
 class ConfigOption(TypedDict):
     plugin: str
     type: str | list[str]
@@ -44,39 +76,23 @@ class KeyValueWidget(forms.Widget):
     def _get_config_options(self) -> dict[str, ConfigOption]:
         """Get available config options from plugins."""
         try:
-            from archivebox.config.common import ArchiveBoxConfig
-            from archivebox.hooks import discover_plugin_configs
+            from archivebox.config.common import config_field_metadata
 
             options: dict[str, ConfigOption] = {}
-            skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "CRAWL_OUTPUT_DIR", "SNAP_DIR"}
-            for key, field in ArchiveBoxConfig.model_fields.items():
-                if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys:
-                    continue
-                default = field.default
-                try:
-                    json.dumps(default)
-                except TypeError:
-                    default = str(default)
-                options[key] = {
-                    "plugin": "archivebox",
-                    "type": str(field.annotation),
-                    "default": default,
-                    "description": field.description or "",
+            for key, metadata in config_field_metadata().items():
+                option_type = metadata.get("type", "string")
+                option: ConfigOption = {
+                    "plugin": str(metadata.get("plugin", "archivebox")),
+                    "type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
+                    "default": metadata.get("default", ""),
+                    "description": str(metadata.get("description", "")),
                 }
-
-            plugin_configs = discover_plugin_configs()
-            for plugin_name, schema in plugin_configs.items():
-                for key, prop in schema.get("properties", {}).items():
-                    option: ConfigOption = {
-                        "plugin": plugin_name,
-                        "type": prop.get("type", "string"),
-                        "default": prop.get("default", ""),
-                        "description": prop.get("description", ""),
-                    }
+                schema = metadata.get("schema")
+                if isinstance(schema, Mapping):
                     for schema_key in ("enum", "pattern", "minimum", "maximum"):
-                        if schema_key in prop:
-                            option[schema_key] = prop[schema_key]
-                    options[key] = option
+                        if schema_key in schema:
+                            option[schema_key] = schema[schema_key]
+                options[key] = option
             return options
         except Exception:
             return {}
@@ -117,7 +133,7 @@ class KeyValueWidget(forms.Widget):
         config_meta_json = json.dumps(config_options)
 
         html = f'''
-        <div id="{widget_id}_container" class="key-value-editor" style="max-width: 700px;">
+        <div id="{widget_id}_container" class="key-value-editor" style="width: 100%; max-width: none;">
             <datalist id="{widget_id}_keys">
                 {datalist_options}
             </datalist>
@@ -227,9 +243,6 @@ class KeyValueWidget(forms.Widget):
                             return 'Example: ["value"]';
                         }}
                         if (types.includes('object')) {{
-                            if (key === 'SAVE_ALLOWLIST' || key === 'SAVE_DENYLIST') {{
-                                return 'Example: {{"^https://example\\\\.com": ["wget"]}}';
-                            }}
                             return 'Example: {{"key": "value"}}';
                         }}
                         return '';
@@ -238,8 +251,6 @@ class KeyValueWidget(forms.Widget):
                     function isRegexConfigKey_{widget_id}(key) {{
                         return key === 'URL_ALLOWLIST' ||
                             key === 'URL_DENYLIST' ||
-                            key === 'SAVE_ALLOWLIST' ||
-                            key === 'SAVE_DENYLIST' ||
                             key.endsWith('_PATTERN') ||
                             key.includes('REGEX');
                     }}
@@ -606,7 +617,7 @@ class KeyValueWidget(forms.Widget):
                         var newRow = document.createElement('div');
                         newRow.className = 'key-value-row';
                         newRow.style.cssText = 'margin-bottom: 6px;';
-                        newRow.innerHTML = '<div style="display: flex; gap: 8px; align-items: center;">' +
+                        newRow.innerHTML = '<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">' +
                             '<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
                             'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
                             '<input type="text" class="kv-value" placeholder="value" ' +
@@ -633,17 +644,74 @@ class KeyValueWidget(forms.Widget):
 
                     window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
 
+                    function focusConfigKeyFromHash_{widget_id}() {{
+                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
+                        // to (or creates) the matching row in this editor. Used by
+                        // the in-banner "pin via admin" link and the
+                        // ``→ Edit <KEY> in Machine.config`` shortcut on the live
+                        // config detail page.
+                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
+                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
+                            return;
+                        }}
+                        var container = document.getElementById('{widget_id}_rows');
+                        if (!container) {{
+                            return;
+                        }}
+                        var match = null;
+                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
+                            if (match) {{ return; }}
+                            var keyInput = row.querySelector('.kv-key');
+                            if (keyInput && keyInput.value.trim() === hash) {{
+                                match = row;
+                            }}
+                        }});
+                        if (!match) {{
+                            // No existing row for this key — prepopulate one with the
+                            // key filled in but value left blank so the operator just
+                            // types/pastes the value and hits save.
+                            window.addKeyValueRow_{widget_id}();
+                            var rows = container.querySelectorAll('.key-value-row');
+                            match = rows[rows.length - 1];
+                            var keyInput = match.querySelector('.kv-key');
+                            if (keyInput) {{
+                                keyInput.value = hash;
+                                keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
+                            }}
+                        }}
+                        if (!match) {{
+                            return;
+                        }}
+                        match.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
+                        var prevOutline = match.style.outline;
+                        match.style.outline = '2px solid #f59e0b';
+                        match.style.outlineOffset = '2px';
+                        match.style.transition = 'outline 1.2s ease-out';
+                        setTimeout(function() {{
+                            match.style.outline = prevOutline || 'none';
+                        }}, 1400);
+                        var valueInput = match.querySelector('.kv-value');
+                        if (valueInput) {{
+                            valueInput.focus();
+                            try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}}
+                        }}
+                    }}
+
                     // Initialize on load
                     document.addEventListener('DOMContentLoaded', function() {{
                         initializeRows_{widget_id}();
                         updateHiddenField_{widget_id}();
+                        focusConfigKeyFromHash_{widget_id}();
                     }});
                     // Also run immediately in case DOM is already ready
                     if (document.readyState !== 'loading') {{
                         initializeRows_{widget_id}();
                         updateHiddenField_{widget_id}();
+                        focusConfigKeyFromHash_{widget_id}();
                     }}
 
+                    window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id});
+
                     // Update on any input change
                     var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
 
@@ -670,12 +738,34 @@ class KeyValueWidget(forms.Widget):
         return mark_safe(html)
 
     def _render_row(self, widget_id: str, key: str, value: str) -> str:
+        from archivebox.config.common import is_sensitive_config_key
+
+        # Sensitive keys (``*TOKEN*``, ``*SECRET*``, ``*API_KEY*``, ``*APIKEY*``) are
+        # rendered write-only: the input is a password field with a placeholder
+        # showing the value is set, but the raw value is NEVER sent to the browser.
+        # When the user submits the form with the field left blank, the
+        # ``ConfigEditorMixin.save_model`` hook re-merges the previously-saved
+        # value so leaving it untouched is a no-op rather than a destructive clear.
+        is_sensitive = is_sensitive_config_key(key)
+        has_value = bool(value)
+        if is_sensitive:
+            input_type = "password"
+            rendered_value = ""
+            placeholder = (
+                "•••••• (saved — enter new value to replace, clear by deleting row)" if has_value else "value (will be saved write-only)"
+            )
+            extra_attrs = ' autocomplete="off" data-sensitive="1"' + (' data-had-value="1"' if has_value else "")
+        else:
+            input_type = "text"
+            rendered_value = self._escape(value)
+            placeholder = "value"
+            extra_attrs = ""
         return f'''
             <div class="key-value-row" style="margin-bottom: 6px;">
-                <div style="display: flex; gap: 8px; align-items: center;">
+                <div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">
                     <input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
                            style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <input type="text" class="kv-value" value="{self._escape(value)}" placeholder="value"
+                    <input type="{input_type}" class="kv-value" value="{rendered_value}" placeholder="{self._escape(placeholder)}"{extra_attrs}
                            style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
                     <datalist class="kv-value-options"></datalist>
                     <button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
@@ -710,7 +800,7 @@ class ConfigEditorMixin(admin.ModelAdmin):
 
     def formfield_for_dbfield(
         self,
-        db_field: models.Field[object, object],
+        db_field: models.Field,
         request: HttpRequest,
         **kwargs: object,
     ) -> forms.Field | None:
@@ -719,6 +809,37 @@ class ConfigEditorMixin(admin.ModelAdmin):
             kwargs["widget"] = KeyValueWidget()
         return super().formfield_for_dbfield(db_field, request, **kwargs)
 
+    def save_model(self, request: HttpRequest, obj, form, change):
+        """Preserve write-only redacted credentials on save.
+
+        The KeyValueWidget renders sensitive keys (``*TOKEN*``, ``*SECRET*``,
+        ``*API_KEY*``, ``*APIKEY*``) with an empty value + password input —
+        the real value never leaves the server. On submit, an empty value
+        for a sensitive key that was previously set means "leave untouched",
+        not "clear it." We honor that here by re-merging the stored value
+        before the row is written. Explicitly removing the row in the UI
+        still clears it (the key is gone from the submitted JSON, so there's
+        nothing to merge over).
+        """
+        from archivebox.config.common import is_sensitive_config_key
+
+        if change and obj.pk and obj.config is not None:
+            try:
+                stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
+            except Exception:
+                stored = {}
+            if isinstance(stored, dict):
+                new_config = dict(obj.config or {})
+                for key, new_value in list(new_config.items()):
+                    if not is_sensitive_config_key(key):
+                        continue
+                    if new_value not in (None, "") and new_value != "********":
+                        continue
+                    if key in stored:
+                        new_config[key] = stored[key]
+                obj.config = new_config
+        super().save_model(request, obj, form, change)
+
 
 class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
     list_display = ("id", "created_at", "created_by")
@@ -741,3 +862,30 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
         if "created_by" in form.base_fields:
             form.base_fields["created_by"].initial = request.user
         return form
+
+    def get_urls(self):
+        """Swap the per-object admin URLs from ``<path:object_id>`` to
+        ``<hexuuid:object_id>`` so canonical change/delete/history URLs use the
+        32-char hex form. The hyphenated form still resolves because the
+        converter's regex accepts both — Django reverses through ``to_url``
+        which always emits hex, so links in templates / changelists / inline
+        formsets all canonicalize automatically.
+
+        Non-UUID PKs (an ``IntegerField`` PK on some legacy table, for example)
+        won't match the converter's regex and fall back to the default
+        ``<path:object_id>`` patterns we still include after our swap.
+        """
+        info = self.opts.app_label, self.opts.model_name
+        object_routes = [
+            path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="%s_%s_history" % info),
+            path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="%s_%s_delete" % info),
+            path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="%s_%s_change" % info),
+        ]
+        # Append after super().get_urls() so our patterns are the
+        # *last-registered* ones with the canonical admin URL names — Django's
+        # reverse() picks the later registration when names collide, which is
+        # how we make ``reverse("admin:app_model_change", args=[obj.pk])``
+        # emit the hex form. The original ``<path:object_id>`` routes stay in
+        # place as a fallback for non-UUID PKs and for resolving inbound
+        # hyphenated URLs (the ``hexuuid`` regex accepts both forms anyway).
+        return super().get_urls() + object_routes

archivebox/base_models/models.py

@@ -2,12 +2,17 @@
 
 __package__ = "archivebox.base_models"
 
-from archivebox.uuid_compat import uuid7
+import json
+import shutil
+from typing import Any
+
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
 
 from django.db import models
 from django.db.models import F
 from django.db import transaction
+from django.db.models.signals import pre_delete
 from django.utils import timezone
 from django.contrib.auth import get_user_model
 from django.urls import reverse_lazy
@@ -15,6 +20,27 @@ from django.conf import settings
 
 from django_stubs_ext.db.models import TypedModelMeta
 
+from archivebox.config import CONSTANTS
+
+
+def normalize_config_json_values(config: Any) -> Any:
+    if not isinstance(config, dict):
+        return config
+
+    normalized = dict(config)
+    for key, value in list(normalized.items()):
+        if not isinstance(value, str) or len(value) < 2:
+            continue
+        if value[:1] != '"' or value[-1:] != '"':
+            continue
+        try:
+            decoded = json.loads(value)
+        except ValueError:
+            continue
+        if isinstance(decoded, str):
+            normalized[key] = decoded
+    return normalized
+
 
 def get_or_create_system_user_pk(username="system"):
     User = get_user_model()
@@ -33,7 +59,7 @@ class AutoDateTimeField(models.DateTimeField):
     """DateTimeField that automatically updates on save (legacy compatibility)."""
 
     def pre_save(self, model_instance, add):
-        if add or not getattr(model_instance, self.attname):
+        if add or self.attname not in model_instance.__dict__ or not model_instance.__dict__[self.attname]:
             value = timezone.now()
             setattr(model_instance, self.attname, value)
             return value
@@ -41,7 +67,7 @@ class AutoDateTimeField(models.DateTimeField):
 
 
 class ModelWithUUID(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
     created_by = models.ForeignKey(
@@ -89,6 +115,10 @@ class ModelWithHealthStats(models.Model):
     class Meta(TypedModelMeta):
         abstract = True
 
+    @property
+    def admin_change_url(self) -> str:
+        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
+
     @property
     def health(self) -> int:
         total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
@@ -97,7 +127,12 @@ class ModelWithHealthStats(models.Model):
     def increment_health_stats(self, success: bool):
         """Atomically increment success or failure counter using F() expression."""
         field = "num_uses_succeeded" if success else "num_uses_failed"
-        type(self).objects.filter(pk=self.pk).update(**{field: F(field) + 1})
+        type(self).objects.filter(pk=self.pk).update(
+            **{
+                field: F(field) + 1,
+                "modified_at": timezone.now(),
+            },
+        )
 
 
 class ModelWithConfig(models.Model):
@@ -108,11 +143,94 @@ class ModelWithConfig(models.Model):
     class Meta(TypedModelMeta):
         abstract = True
 
+    def save(self, *args, **kwargs):
+        normalized_config = normalize_config_json_values(self.config)
+        if normalized_config != self.config:
+            self.config = normalized_config
+            update_fields = kwargs.get("update_fields")
+            if update_fields is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
+        super().save(*args, **kwargs)
+
+
+class ModelWithDeleteAfter(models.Model):
+    delete_after_final_statuses: tuple[str, ...] = ()
+    delete_at = models.DateTimeField(default=None, null=True, blank=True, db_index=True)
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    def save(self, *args, **kwargs):
+        update_fields = kwargs.get("update_fields")
+        if self.delete_at is None:
+            self.set_delete_at_from_config()
+            if self.delete_at is not None and update_fields is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"]))
+        super().save(*args, **kwargs)
+
+    def get_delete_after_config_value(self):
+        from archivebox.config.common import get_config
+
+        return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER
+
+    def set_delete_at_from_config(self, config_value=None) -> bool:
+        if self.delete_at is not None:
+            return False
+
+        from archivebox.config.common import parse_delete_after
+
+        duration = parse_delete_after(self.get_delete_after_config_value() if config_value is None else config_value)
+        if duration is None:
+            return False
+
+        self.delete_at = (self.created_at or timezone.now()) + duration
+        return True
+
+    @classmethod
+    def missing_delete_at_candidates(cls):
+        return cls.objects.none()
+
+    @classmethod
+    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
+        if backfill_missing:
+            missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
+            for obj in missing_delete_at:
+                if obj.set_delete_at_from_config():
+                    cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
+                        delete_at=obj.delete_at,
+                        modified_at=timezone.now(),
+                    )
+
+        # Keep the expiration sweep anchored on delete_at. Some large tables
+        # have millions of final-status rows but almost no retained rows; adding
+        # the status filter before SQLite has narrowed by delete_at can make the
+        # runner scan the hot status index before it claims new work.
+        due_pks = list(
+            cls.objects.filter(delete_at__isnull=False, delete_at__lte=timezone.now())
+            .order_by("delete_at", "pk")
+            .values_list("pk", flat=True)[:batch_size],
+        )
+        if not due_pks:
+            return 0
+
+        queryset = cls.objects.filter(pk__in=due_pks)
+        if cls.delete_after_final_statuses:
+            queryset = queryset.filter(status__in=cls.delete_after_final_statuses)
+
+        count = 0
+        expired = list(queryset.order_by("delete_at", "pk"))
+        for obj in expired:
+            obj.delete()
+            count += 1
+        return count
+
 
 class ModelWithOutputDir(ModelWithUUID):
     class Meta(ModelWithUUID.Meta):
         abstract = True
 
+    _delete_signal_registered = False
+
     def save(self, *args, **kwargs):
         super().save(*args, **kwargs)
         output_dir = Path(self.output_dir)
@@ -135,3 +253,51 @@ class ModelWithOutputDir(ModelWithUUID):
     @property
     def output_dir(self) -> Path:
         raise NotImplementedError(f"{self.__class__.__name__} must implement output_dir property")
+
+    def output_paths_for_delete(self) -> tuple[Path, ...]:
+        return (Path(self.output_dir),)
+
+    @classmethod
+    def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
+        data_dir = CONSTANTS.DATA_DIR.resolve()
+        safe_paths = []
+        for raw_path in paths:
+            path = Path(raw_path)
+            is_safe = False
+            for candidate in (path.absolute(), path.resolve()):
+                try:
+                    candidate.relative_to(data_dir)
+                    is_safe = True
+                    break
+                except ValueError:
+                    continue
+            if not is_safe:
+                raise ValueError(f"Refusing to delete output path outside DATA_DIR: {path}")
+            safe_paths.append(path)
+        return tuple(safe_paths)
+
+    @classmethod
+    def delete_output_paths(cls, paths) -> None:
+        for path in cls.validate_output_paths_for_delete(paths):
+            if path.is_symlink() or path.is_file():
+                path.unlink(missing_ok=True)
+            elif path.is_dir():
+                shutil.rmtree(path, ignore_errors=True)
+
+    @classmethod
+    def register_delete_signal(cls) -> None:
+        if cls._delete_signal_registered:
+            return
+
+        def schedule_output_dir_cleanup(sender, instance, **kwargs):
+            if not isinstance(instance, ModelWithOutputDir):
+                return
+            paths = instance.validate_output_paths_for_delete(instance.output_paths_for_delete())
+            transaction.on_commit(lambda paths=paths: instance.delete_output_paths(paths))
+
+        pre_delete.connect(
+            schedule_output_dir_cleanup,
+            dispatch_uid="archivebox.output_dir_cleanup_on_delete",
+            weak=False,
+        )
+        cls._delete_signal_registered = True

archivebox/cli/__init__.py

@@ -5,15 +5,26 @@ import sys
 from importlib import import_module
 
 import rich_click as click
-from rich import print
+from rich.console import Console
 
 from archivebox.config.version import VERSION
 
+STDERR = Console(stderr=True)
+
 
 if "--debug" in sys.argv:
     os.environ["DEBUG"] = "True"
     sys.argv.remove("--debug")
 
+# Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
+# `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
+# the subcommand executes. `--quick-init` is the old server spelling. Strip either from
+# argv here so each subcommand's own click parser never sees it. Ignored for `help` and
+# `init` themselves.
+if "--init" in sys.argv or "--quick-init" in sys.argv:
+    sys.argv = [arg for arg in sys.argv if arg not in ("--init", "--quick-init")]
+    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+
 
 class ArchiveBoxGroup(click.Group):
     """lazy loading click group for archivebox commands"""
@@ -22,6 +33,7 @@ class ArchiveBoxGroup(click.Group):
         "help": "archivebox.cli.archivebox_help.main",
         "version": "archivebox.cli.archivebox_version.main",
         "mcp": "archivebox.cli.archivebox_mcp.main",
+        "oneshot": "archivebox.cli.archivebox_oneshot.main",
     }
     setup_commands = {
         "init": "archivebox.cli.archivebox_init.main",
@@ -56,10 +68,6 @@ class ArchiveBoxGroup(click.Group):
         # Introspection commands
         "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
     }
-    legacy_model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl_compat.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot_compat.main",
-    }
     all_subcommands = {
         **meta_commands,
         **setup_commands,
@@ -71,51 +79,43 @@ class ArchiveBoxGroup(click.Group):
         "import": "add",
         "archive": "add",
     }
-    legacy_model_subcommands = {
-        "crawl": {"create", "list", "update", "delete"},
-        "snapshot": {"create", "list", "update", "delete"},
-    }
 
     @classmethod
     def get_canonical_name(cls, cmd_name):
         return cls.renamed_commands.get(cmd_name, cmd_name)
 
     @classmethod
-    def _should_use_legacy_model_command(cls, cmd_name: str) -> bool:
-        if cmd_name not in cls.legacy_model_commands:
-            return False
+    def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool:
+        wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
+        return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands)
 
-        try:
-            arg_idx = sys.argv.index(cmd_name)
-        except ValueError:
-            return False
+    @classmethod
+    def _setup_django_for_lazy_import(cls, cmd_name: str) -> None:
+        if not cls._needs_django_for_lazy_import(cmd_name):
+            return
 
-        remaining_args = sys.argv[arg_idx + 1 :]
-        if not remaining_args:
-            return False
+        from django.apps import apps
 
-        first_arg = remaining_args[0]
-        if first_arg in ("-h", "--help"):
-            return False
+        if apps.ready:
+            return
 
-        return first_arg not in cls.legacy_model_subcommands[cmd_name]
+        from archivebox.config.django import setup_django
+
+        setup_django()
 
     def get_command(self, ctx, cmd_name):
         # handle renamed commands
         if cmd_name in self.renamed_commands:
             new_name = self.renamed_commands[cmd_name]
-            print(
+            STDERR.print(
                 f" [violet]Hint:[/violet] `archivebox {cmd_name}` has been renamed to `archivebox {new_name}`",
-                file=sys.stderr,
             )
             cmd_name = new_name
             ctx.invoked_subcommand = cmd_name
 
-        if self._should_use_legacy_model_command(cmd_name):
-            return self._lazy_load(self.legacy_model_commands[cmd_name])
-
         # handle lazy loading of commands
         if cmd_name in self.all_subcommands:
+            self._setup_django_for_lazy_import(cmd_name)
             return self._lazy_load(cmd_name)
 
         # fall-back to using click's default command lookup
@@ -130,9 +130,9 @@ class ArchiveBoxGroup(click.Group):
 
         # print(f'LAZY LOADING {import_path}')
         mod = import_module(modname)
-        func = getattr(mod, funcname)
+        func = vars(mod)[funcname]
 
-        if not hasattr(func, "__doc__"):
+        if func.__doc__ is None:
             raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
 
         # if not isinstance(cmd, click.BaseCommand):
@@ -156,7 +156,8 @@ def cli(ctx, help=False):
 
     # if the subcommand is in archive_commands or model_commands,
     # then we need to set up the django environment and check that we're in a valid data folder
-    if subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands:
+    wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
+    if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
         # print('SETUP DJANGO AND CHECK DATA FOLDER')
         try:
             if subcommand == "server":
@@ -165,34 +166,59 @@ def cli(ctx, help=False):
                     os.environ["ARCHIVEBOX_RUNSERVER"] = "1"
                     if "--reload" in sys.argv:
                         os.environ["ARCHIVEBOX_AUTORELOAD"] = "1"
-                        from archivebox.config.common import get_config
-
-                        os.environ["ARCHIVEBOX_RUNSERVER_PIDFILE"] = str(get_config().TMP_DIR / "runserver.pid")
 
             from archivebox.config.django import setup_django
-            from archivebox.misc.checks import check_data_folder
+            from archivebox.misc.checks import check_data_folder, check_migrations
 
             setup_django()
+            if os.environ.get("ARCHIVEBOX_WANTS_INIT") == "1" and subcommand not in ("init", "help"):
+                # Universal `--init` was passed: build/upgrade the data folder before
+                # the regular preflight runs, so it succeeds on a fresh dir and an
+                # out-of-date schema both. Drop the env var afterwards so spawned
+                # subprocesses (supervisord workers, daphne, runner, etc.) inherit
+                # a clean env and don't re-trigger init in every child.
+                from archivebox.cli.archivebox_init import init as archivebox_init
+
+                archivebox_init(quick=True)
+                os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
             check_data_folder()
+            if subcommand != "update":
+                check_migrations(auto_apply=True)
         except Exception as e:
-            print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]", file=sys.stderr)
+            STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
             if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
                 raise
 
 
-def main(args=None, prog_name=None, stdin=None):
+def main(args=None, prog_name=None):
     # show `docker run archivebox xyz` in help messages if running in docker
     IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
     IS_TTY = sys.stdin.isatty()
     prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")
 
-    # stdin param allows passing input data from caller (used by __main__.py)
-    # currently not used by click-based CLI, but kept for backwards compatibility
+    previous_unraisablehook = sys.unraisablehook
 
+    def ignore_shutdown_unraisable(unraisable):
+        if isinstance(unraisable.exc_value, (KeyboardInterrupt, SystemExit)):
+            return
+        previous_unraisablehook(unraisable)
+
+    sys.unraisablehook = ignore_shutdown_unraisable
     try:
-        cli(args=args, prog_name=prog_name)
+        cli(args=args, prog_name=prog_name, standalone_mode=False)
+    except click.Abort:
+        STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
+        raise SystemExit(130) from None
+    except click.ClickException as err:
+        err.show()
+        raise SystemExit(err.exit_code) from None
+    except click.exceptions.Exit as err:
+        raise SystemExit(err.exit_code) from None
     except KeyboardInterrupt:
-        print("\n\n[red][X] Got CTRL+C. Exiting...[/red]")
+        STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
+        raise SystemExit(130) from None
+    finally:
+        sys.unraisablehook = previous_unraisablehook
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_add.py

@@ -1,46 +1,51 @@
 #!/usr/bin/env python3
+from __future__ import annotations
 
 __package__ = "archivebox.cli"
 __command__ = "archivebox add"
 
 import sys
+import os
 import json
 from pathlib import Path
 
-from typing import TYPE_CHECKING
+from typing import Any, TYPE_CHECKING
 
 import rich_click as click
 
-from django.utils import timezone
-from django.db.models import QuerySet
-
+from archivebox.config import CONSTANTS
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.util import parse_filesize_to_bytes
-from archivebox import CONSTANTS
-from archivebox.config.permissions import USER, HOSTNAME
-from archivebox.config.common import get_config
 
 
 if TYPE_CHECKING:
+    from django.db.models import QuerySet
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import Snapshot
 
 
-def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
+def _collect_input_urls(args: tuple[str, ...], *, parser: str = "auto") -> list[str]:
     from archivebox.misc.jsonl import read_args_or_stdin
+    from archivebox.misc.util import validate_url
 
     urls: list[str] = []
     for record in read_args_or_stdin(args):
         url = record.get("url")
         if isinstance(url, str) and url:
-            urls.append(url)
+            try:
+                urls.append(validate_url(url))
+            except ValueError as err:
+                raise click.BadParameter(str(err), param_hint="URL") from err
 
         urls_field = record.get("urls")
         if isinstance(urls_field, str):
             for line in urls_field.splitlines():
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    urls.append(line)
+                    try:
+                        urls.append(validate_url(line))
+                    except ValueError as err:
+                        raise click.BadParameter(str(err), param_hint="URL") from err
 
     return urls
 
@@ -52,6 +57,7 @@ def add(
     depth: int | str = 0,
     max_urls: int = 0,
     crawl_max_size: int | str = 0,
+    crawl_timeout: int = 0,
     snapshot_max_size: int | str = 0,
     crawl_max_concurrent_snapshots: int | None = None,
     tag: str = "",
@@ -60,12 +66,11 @@ def add(
     parser: str = "auto",
     plugins: str = "",
     persona: str = "Default",
-    overwrite: bool = False,
-    update: bool | None = None,
     index_only: bool = False,
     bg: bool = False,
     created_by_id: int | None = None,
-) -> tuple["Crawl", QuerySet["Snapshot"]]:
+    config: dict[str, Any] | None = None,
+) -> tuple[Crawl, QuerySet[Snapshot]]:
     """Add a new URL or list of URLs to your archive.
 
     The flow is:
@@ -83,10 +88,16 @@ def add(
     depth = int(depth)
     max_urls = int(max_urls or 0)
     crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
+    crawl_timeout = int(crawl_timeout or 0)
     snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
-    config = get_config()
+    from archivebox.config.permissions import USER, HOSTNAME
+    from archivebox.config.common import get_config
+
+    config_overrides = dict(config or {})
+    runtime_config = get_config()
+    crawl_max_concurrent_snapshots_override = crawl_max_concurrent_snapshots is not None
     if crawl_max_concurrent_snapshots is None:
-        crawl_max_concurrent_snapshots = config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
+        crawl_max_concurrent_snapshots = runtime_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
     crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)
 
     if depth not in (0, 1, 2, 3, 4):
@@ -95,6 +106,8 @@ def add(
         raise ValueError("max_urls must be >= 0")
     if crawl_max_size < 0:
         raise ValueError("crawl_max_size must be >= 0")
+    if crawl_timeout < 0:
+        raise ValueError("crawl_timeout must be >= 0")
     if snapshot_max_size < 0:
         raise ValueError("snapshot_max_size must be >= 0")
     if crawl_max_concurrent_snapshots < 1:
@@ -106,29 +119,18 @@ def add(
     from archivebox.personas.models import Persona
     from archivebox.misc.logging_util import printable_filesize
     from archivebox.misc.system import get_dir_size
-    from archivebox.services.runner import run_crawl
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    from django.utils import timezone
 
     created_by_id = created_by_id or get_or_create_system_user_pk()
     started_at = timezone.now()
-    if update is None:
-        update = not config.ONLY_NEW
 
-    if isinstance(urls, str):
-        url_list = [line.strip() for line in urls.splitlines() if line.strip()]
-    else:
-        url_list = [str(url).strip() for url in urls if str(url).strip()]
-    if snapshot_ids and len(snapshot_ids) != len(url_list):
-        raise ValueError("snapshot_ids length must match urls length")
-
-    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
-    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
-    sources_file.parent.mkdir(parents=True, exist_ok=True)
-    if snapshot_ids:
-        sources_file.write_text(
-            "\n".join(json.dumps({"url": url, "id": snapshot_ids[index], "tags": tag, "depth": 0}) for index, url in enumerate(url_list)),
-        )
-    else:
-        sources_file.write_text("\n".join(url_list))
+    use_internal_input_root = isinstance(urls, str)
+    source_text = (
+        urls
+        if use_internal_input_root
+        else "\n".join(json.dumps({"type": "CrawlSeed", "url": str(url), "depth": 0}, separators=(",", ":")) for url in urls)
+    )
 
     # 2. Create a new Crawl with inline URLs
     cli_args = [*sys.argv]
@@ -138,36 +140,54 @@ def add(
 
     timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
 
-    # Read URLs directly into crawl
-    urls_content = sources_file.read_text()
     persona_name = (persona or "Default").strip() or "Default"
-    plugins = plugins or str(config.get("PLUGINS") or "")
-    persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
+    plugins = plugins or ""
+    persona_obj = Persona.get_or_create_named(persona_name)
     persona_obj.ensure_dirs()
+    effective_persona_config = get_config(persona=persona_obj)
 
+    crawl_config = {
+        "PERMISSIONS": str(effective_persona_config.PERMISSIONS),
+        **({"INDEX_ONLY": True} if index_only else {}),
+        **({"PLUGINS": plugins} if plugins else {}),
+        **(
+            {"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots}
+            if crawl_max_concurrent_snapshots_override
+            and crawl_max_concurrent_snapshots != int(effective_persona_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS)
+            else {}
+        ),
+        **({"CRAWL_MAX_URLS": max_urls} if max_urls else {}),
+        **({"CRAWL_MAX_SIZE": crawl_max_size} if crawl_max_size else {}),
+        **({"CRAWL_TIMEOUT": crawl_timeout} if crawl_timeout else {}),
+        **({"SNAPSHOT_MAX_SIZE": snapshot_max_size} if snapshot_max_size else {}),
+        **({"PARSER": parser} if parser != "auto" else {}),
+        **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
+        **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
+    }
+    # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
+    # priority for crawl-frozen keys. Runtime-derived execution keys are
+    # stripped by Crawl.save() and rederived when hooks run.
+    crawl_config.update(config_overrides)
+
+    # Crawl.urls is the user's submitted source, not a derived work queue for
+    # this add path. Keeping it byte-for-byte readable is what lets API/UI/CLI
+    # callers audit or resume imports without losing RSS/Netscape/JSON metadata
+    # that is not representable as one plain URL per line.
     crawl = Crawl.objects.create(
-        urls=urls_content,
-        max_depth=depth,
-        max_urls=max_urls,
-        crawl_max_size=crawl_max_size,
-        snapshot_max_size=snapshot_max_size,
+        urls=source_text,
+        # Stdin/import text gets an extra hop because the synthetic
+        # archivebox://internal root lives at depth 0 and parser-discovered
+        # URLs land at depth 1; direct URL args become the depth=0 input
+        # snapshots themselves so --depth=N matches the deepest hop the user
+        # asked for.
+        max_depth=depth + 1 if use_internal_input_root else depth,
         tags_str=tag,
         persona_id=persona_obj.id,
         label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
         created_by_id=created_by_id,
-        status=Crawl.StatusChoices.QUEUED if bg or index_only else Crawl.StatusChoices.STARTED,
-        retry_at=timezone.now() if bg else None,
-        config={
-            "ONLY_NEW": not update,
-            "INDEX_ONLY": index_only,
-            "OVERWRITE": overwrite,
-            "PLUGINS": plugins,
-            "DEFAULT_PERSONA": persona_name,
-            "CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots,
-            "PARSER": parser,
-            **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
-            **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
-        },
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at=None if index_only else timezone.now(),
+        config=crawl_config,
     )
 
     print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
@@ -179,13 +199,8 @@ def add(
     #    Discovered URLs become child Snapshots (depth+1)
 
     if index_only:
-        # Just create the crawl but don't start processing
-        print("[yellow]\\[*] Index-only mode - crawl created but not started[/yellow]")
-        crawl.create_snapshots_from_urls()
-        return crawl, crawl.snapshot_set.all()
-
-    if bg:
-        crawl.create_snapshots_from_urls()
+        print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
+        return crawl, crawl.snapshot_set.none()
 
     # 5. Start the crawl runner to process the queue
     #    The runner will:
@@ -195,25 +210,70 @@ def add(
     #    - Repeat until max_depth reached
 
     if bg:
-        # Background mode: just queue work and return (background runner via server will pick it up)
+        # Background mode: just queue work and return (background runner via server will pick it up).
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
+        from archivebox.services.runner import ensure_background_runner
+
+        ensure_background_runner(allow_under_pytest=True)
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
-        run_crawl(str(crawl.id))
+        from archivebox.machine.models import Process
+        from archivebox.core.takeover_util import command_owns_foreground_runner, current_command, standby_until_foreground_runner_needed
+        from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process
+
+        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
+        exit_code = 0
+        try:
+            try:
+                with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
+                    while True:
+                        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                        exit_code = run_runner_worker(
+                            ["--crawl-id", str(crawl.id)],
+                            name=f"worker_runner_add_{os.getpid()}",
+                            interactive_interrupts=True,
+                        )
+                        crawl.refresh_from_db(fields=["status", "retry_at"])
+                        if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
+                            break
+                        # A shared supervisord can be stopped by another
+                        # foreground owner (for example `archivebox server`
+                        # shutting down) and stop the one-shot add runner while
+                        # this add's crawl is still runnable. Keep the add
+                        # process alive so it can re-enter the normal
+                        # foreground-runner ownership loop and finish its crawl.
+                        supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None
+                        if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone):
+                            continue
+                        if exit_code == 0:
+                            break
+                        if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
+                            continue
+                        raise SystemExit(exit_code)
+            except KeyboardInterrupt:
+                exit_code = 130
+                print("\n[red][X] archivebox add interrupted.[/red]")
+                print("[yellow]Hint: resume this crawl with:[/yellow]")
+                print(f"    [green]archivebox run --crawl-id={crawl.id}[/green]")
+                raise SystemExit(exit_code)
+        finally:
+            command.mark_exited(exit_code=exit_code)
+            stop_own_supervisord_process()
 
         # Print summary for foreground runs
         try:
             crawl.refresh_from_db()
-            snapshots_count = crawl.snapshot_set.count()
             try:
                 from django.db.models import Count, Sum
 
-                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("archiveresult__output_size"))
-                total_bytes = int(totals["total_bytes"] or 0) if totals["snapshot_count"] else 0
+                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("output_size"))
+                snapshots_count = int(totals["snapshot_count"] or 0)
+                total_bytes = int(totals["total_bytes"] or 0)
             except Exception:
+                snapshots_count = crawl.snapshot_set.count()
                 total_bytes, _, _ = get_dir_size(crawl.output_dir)
             total_size = printable_filesize(total_bytes)
             total_time = timezone.now() - started_at
@@ -234,12 +294,9 @@ def add(
             except Exception:
                 rel_output_str = str(crawl.output_dir)
 
-            bind_addr = config.BIND_ADDR or "127.0.0.1:8000"
-            if bind_addr.startswith("http://") or bind_addr.startswith("https://"):
-                base_url = bind_addr
-            else:
-                base_url = f"http://{bind_addr}"
-            admin_url = f"{base_url}/admin/crawls/crawl/{crawl.id}/change/"
+            from archivebox.core.routes_util import build_admin_url
+
+            admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
 
             print("\n[bold]crawl output saved to:[/bold]")
             print(f"  {rel_output_str}")
@@ -266,6 +323,7 @@ def add(
 )
 @click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)")
 @click.option("--crawl-max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
+@click.option("--crawl-timeout", type=int, default=0, help="Maximum total crawl runtime in seconds (0 = unlimited)")
 @click.option("--snapshot-max-size", default="0", help="Maximum per-snapshot size in bytes or units like 45mb / 1gb (0 = unlimited)")
 @click.option("--crawl-max-concurrent-snapshots", type=int, default=None, help="Maximum snapshots to archive concurrently within one crawl")
 @click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3")
@@ -273,34 +331,75 @@ def add(
 @click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
 @click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
 @click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
+@click.option("--extract", default="", help="Comma-separated plugins or output types to extract e.g. title,pdf,text/html,image,...")
 @click.option("--persona", default="Default", help="Authentication profile to use when archiving")
-@click.option("--overwrite", "-F", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
-@click.option("--update", is_flag=True, default=None, help="Retry any previously skipped/failed URLs when re-adding them")
+@click.option(
+    "--only-new/--no-only-new",
+    "only_new",
+    default=None,
+    help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
+    "Pass --no-only-new to force re-archive of URLs that already exist.",
+)
 @click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
+@click.option("--overwrite", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
+@click.option("--update", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
 @click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
 @click.argument("urls", nargs=-1, type=click.Path())
 @docstring(add.__doc__)
 def main(**kwargs):
     """Add a new URL or list of URLs to your archive"""
 
-    raw_urls = kwargs.pop("urls")
-    urls = _collect_input_urls(raw_urls)
-    if not urls:
-        raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
-    if int(kwargs.get("max_urls") or 0) < 0:
-        raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
-    try:
-        kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size"))
-    except ValueError as err:
-        raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err
-    try:
-        kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
-    except ValueError as err:
-        raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
-    if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
-        raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
 
-    add(urls=urls, **kwargs)
+    with foreground_shutdown_signals(), foreground_parent_watchdog():
+        raw_urls = kwargs.pop("urls")
+        if raw_urls:
+            urls = _collect_input_urls(raw_urls, parser=kwargs.get("parser", "auto"))
+        elif not sys.stdin.isatty():
+            urls = sys.stdin.read()
+        else:
+            urls = []
+        if not urls:
+            raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
+        if int(kwargs.get("max_urls") or 0) < 0:
+            raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
+        if int(kwargs.get("crawl_timeout") or 0) < 0:
+            raise click.BadParameter("crawl_timeout must be 0 or a positive integer.", param_hint="--crawl-timeout")
+        try:
+            kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size"))
+        except ValueError as err:
+            raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err
+        try:
+            kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
+        except ValueError as err:
+            raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
+        if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
+            raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
+
+        # Translate --only-new/--no-only-new into a crawl config override.
+        # add() takes config overrides as a dict; no per-flag kwargs.
+        overwrite = kwargs.pop("overwrite", False)
+        update = kwargs.pop("update", False)
+        only_new = kwargs.pop("only_new", None)
+        extract = kwargs.pop("extract", "")
+        if overwrite or update:
+            only_new = False
+        if only_new is not None:
+            kwargs["config"] = {"ONLY_NEW": bool(only_new)}
+        if extract:
+            from abx_dl.models import discover_plugins, plugins_matching_output
+
+            all_plugins = discover_plugins()
+            tokens = [token.strip() for token in extract.split(",") if token.strip()]
+            plugin_names = {name.lower(): name for name in all_plugins}
+            selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
+            selected += plugins_matching_output(all_plugins, tokens)
+            if not selected:
+                raise click.UsageError(f"No plugins found matching extract types: {extract}")
+            existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
+            kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
+
+        add(urls=urls, **kwargs)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_archiveresult.py

@@ -38,7 +38,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict:
@@ -74,7 +74,7 @@ def create_archiveresults(
         1: Failure
     """
     from archivebox.config.common import get_config
-    from archivebox.hooks import discover_hooks
+    from archivebox.plugins.hooks import discover_hooks
     from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
     from archivebox.core.models import Snapshot
 
@@ -152,7 +152,7 @@ def create_archiveresults(
             config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
             hooks = discover_hooks("Snapshot", config=config)
             for hook_path in hooks:
-                hook_name = hook_path.name
+                hook_name = hook_path.stem
                 plugin_name = hook_path.parent.name
                 if not is_tty:
                     write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))

archivebox/cli/archivebox_binary.py

@@ -33,7 +33,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================

archivebox/cli/archivebox_config.py

@@ -6,6 +6,7 @@ import sys
 import toml
 import rich_click as click
 from rich import print
+from pathlib import Path
 
 from archivebox.misc.util import docstring, enforce_types
 from archivebox.misc.toml_util import CustomTOMLEncoder
@@ -30,12 +31,17 @@ def config(
     from archivebox.misc.logging_util import printable_config
     from abx_plugins.plugins.base.utils import resolve_alias
     from archivebox.config.collection import write_config_file
+    from archivebox.config import CONSTANTS_CONFIG
     from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
 
     check_data_folder()
 
     FLAT_CONFIG = get_config().as_dict()
+    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys()
+    readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)}
+    writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config}
+    readable_config = {**writable_config, **readonly_config}
     CONFIGS = get_all_configs()
     plugin_schemas = {
         plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
@@ -56,18 +62,23 @@ def config(
             config_options = [
                 core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
             ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
             for config_section in CONFIGS.values():
-                aliases = getattr(config_section, "aliases", {})
+                aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias}
 
                 for search_key in config_options:
                     # search all aliases in the section
                     for alias_key, key in aliases.items():
-                        if search_key.lower() in alias_key.lower():
+                        if key in readable_config and search_key.lower() in alias_key.lower():
                             matching_config[key] = dict(config_section)[key]
 
                     # search all keys and values in the section
                     for existing_key, value in dict(config_section).items():
+                        if existing_key in readable_config and (
+                            search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower()
+                        ):
+                            matching_config[existing_key] = value
+                    for existing_key, value in readonly_config.items():
                         if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
                             matching_config[existing_key] = value
 
@@ -79,18 +90,18 @@ def config(
             config_options = [
                 core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
             ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
+            failed_config = [key for key in config_options if key not in readable_config]
             if failed_config:
                 print("\n[red][X] These options failed to get[/red]")
                 print("    {}".format("\n    ".join(config_options)))
                 raise SystemExit(1)
         else:
-            matching_config = FLAT_CONFIG
+            matching_config = readable_config
 
         # Display core config sections
         for config_section in CONFIGS.values():
-            section_header = getattr(config_section, "toml_section_header", "")
+            section_header = config_section.toml_section_header
             if isinstance(section_header, str) and section_header:
                 print(f"[grey53]\\[{section_header}][/grey53]")
             else:
@@ -100,12 +111,18 @@ def config(
             print(_format_toml(kv_in_section))
             print("[grey53]################################################################[/grey53]")
 
+        readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config}
+        if readonly_keys:
+            print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
+            print(_format_toml(readonly_keys))
+            print("[grey53]################################################################[/grey53]")
+
         plugin_keys = {}
 
         # Collect all plugin config keys
         for schema in plugin_schemas.values():
             for key in schema.keys():
-                if key in matching_config:
+                if key in matching_config and key in writable_config:
                     plugin_keys[key] = matching_config[key]
 
         # Display all plugin config in single [PLUGINS] section
@@ -135,7 +152,7 @@ def config(
                     f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
                 )
 
-            if key in FLAT_CONFIG:
+            if key in writable_config:
                 new_config[key] = val.strip()
             else:
                 failed_options.append(line)

archivebox/cli/archivebox_crawl.py

@@ -39,7 +39,7 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================
@@ -65,6 +65,7 @@ def create_crawl(
         1: Failure
     """
     from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
+    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
 
@@ -98,7 +99,11 @@ def create_crawl(
         # Collect URLs
         url = record.get("url")
         if url:
-            url_list.append(url)
+            try:
+                url_list.append(validate_url(str(url)))
+            except ValueError as err:
+                rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                return 1
 
         # Handle 'urls' field (newline-separated)
         urls_field = record.get("urls")
@@ -106,7 +111,11 @@ def create_crawl(
             for line in urls_field.split("\n"):
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    url_list.append(line)
+                    try:
+                        url_list.append(validate_url(line))
+                    except ValueError as err:
+                        rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                        return 1
 
     # Output pass-through records first
     if not is_tty:
@@ -247,17 +256,30 @@ def update_crawls(
         try:
             crawl = Crawl.objects.get(id=crawl_id)
 
-            # Apply updates from CLI flags
             if status:
-                crawl.status = status
-                crawl.retry_at = timezone.now()
+                if status not in Crawl.StatusChoices.values:
+                    rprint(f"[red]Invalid crawl status: {status}[/red]", file=sys.stderr)
+                    continue
+                if status == Crawl.StatusChoices.SEALED:
+                    crawl.cancel()
+                elif status == Crawl.StatusChoices.PAUSED:
+                    crawl.pause()
+                elif status == Crawl.StatusChoices.QUEUED:
+                    if crawl.status == Crawl.StatusChoices.PAUSED:
+                        crawl.resume()
+                    else:
+                        crawl.update_and_requeue(status=Crawl.StatusChoices.QUEUED, retry_at=timezone.now())
+                elif status == Crawl.StatusChoices.STARTED:
+                    crawl.update_and_requeue(status=Crawl.StatusChoices.STARTED, retry_at=timezone.now())
             if max_depth is not None:
+                crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
                 crawl.max_depth = max_depth
-
-            crawl.save()
+            elif not status:
+                crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
             updated_count += 1
 
             if not is_tty:
+                crawl.refresh_from_db()
                 write_record(crawl.to_json())
 
         except Crawl.DoesNotExist:

archivebox/cli/archivebox_crawl_compat.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_add import add
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--wait/--no-wait", "wait", default=True, help="Accepted for backwards compatibility")
-@click.argument("urls", nargs=-1)
-def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox crawl URL...` entrypoint."""
-    del status, wait
-    add(list(urls), depth=depth, tag=tag, index_only=True, bg=True)
-    sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_extract.py

@@ -32,6 +32,7 @@ __command__ = "archivebox extract"
 
 import sys
 from collections import defaultdict
+from itertools import product
 
 import rich_click as click
 
@@ -47,10 +48,11 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     from rich import print as rprint
     from django.utils import timezone
     from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.services.runner import run_crawl
 
     try:
-        archiveresult = ArchiveResult.objects.get(id=archiveresult_id)
+        archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
     except ArchiveResult.DoesNotExist:
         rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
         return 1
@@ -58,19 +60,35 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
 
     try:
+        was_paused = archiveresult.snapshot.is_paused
         archiveresult.reset_for_retry()
         snapshot = archiveresult.snapshot
-        snapshot.status = snapshot.StatusChoices.QUEUED
-        snapshot.retry_at = timezone.now()
-        snapshot.save(update_fields=["status", "retry_at", "modified_at"])
-
+        if not was_paused:
+            snapshot.queue_for_extraction()
+        else:
+            # A paused snapshot may still accept explicit maintenance for one
+            # ArchiveResult, but this path must not transition it back to
+            # queued/startable work. Guard: only set retry_at while the row is
+            # still paused — concurrent resume would otherwise see a stale
+            # retry_at marker.
+            snapshot.safe_update(
+                {"retry_at": timezone.now()},
+                refresh=False,
+                extra_filter={"status": snapshot.StatusChoices.PAUSED},
+            )
         crawl = snapshot.crawl
-        if crawl.status != crawl.StatusChoices.STARTED:
-            crawl.status = crawl.StatusChoices.QUEUED
-        crawl.retry_at = timezone.now()
-        crawl.save(update_fields=["status", "retry_at", "modified_at"])
+        if not crawl.claim_processing_lock(lock_seconds=10):
+            rprint(
+                f"[yellow]Crawl {crawl.id} is already owned by another runner[/yellow]",
+                file=sys.stderr,
+            )
+            return 1
 
-        run_crawl(str(crawl.id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
+        try:
+            run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
+        finally:
+            if was_paused:
+                snapshot.restore_paused_scheduler_marker()
         archiveresult.refresh_from_db()
 
         if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
@@ -98,6 +116,8 @@ def run_plugins(
     plugins: str = "",
     wait: bool = True,
     emit_results: bool = True,
+    show_progress: bool = True,
+    preserve_queued: bool = False,
 ) -> int:
     """
     Run plugins on Snapshots from input.
@@ -118,7 +138,9 @@ def run_plugins(
         TYPE_ARCHIVERESULT,
     )
     from archivebox.core.models import Snapshot
+    from archivebox.core.models import ArchiveResult
     from archivebox.services.runner import run_crawl
+    from abx_dl.models import discover_plugins
 
     is_tty = sys.stdout.isatty()
 
@@ -145,7 +167,7 @@ def run_plugins(
         if record_type == TYPE_SNAPSHOT:
             snapshot_id = record.get("id")
             if snapshot_id:
-                snapshot_ids.add(snapshot_id)
+                snapshot_ids.add(str(snapshot_id))
             elif record.get("url"):
                 # Look up by URL (get most recent if multiple exist)
                 snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first()
@@ -157,60 +179,201 @@ def run_plugins(
         elif record_type == TYPE_ARCHIVERESULT:
             snapshot_id = record.get("snapshot_id")
             if snapshot_id:
-                snapshot_ids.add(snapshot_id)
+                snapshot_ids.add(str(snapshot_id))
                 plugin_name = record.get("plugin")
                 if plugin_name and not plugins_list:
                     requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name))
 
         elif "id" in record:
             # Assume it's a snapshot ID
-            snapshot_ids.add(record["id"])
+            snapshot_ids.add(str(record["id"]))
 
     if not snapshot_ids:
         rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr)
         return 1
 
-    # Get snapshots and ensure they have pending ArchiveResults
-    processed_count = 0
-    for snapshot_id in snapshot_ids:
-        try:
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-        except Snapshot.DoesNotExist:
-            rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
-            continue
+    existing_snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids).values_list("id", "crawl_id"))
+    existing_snapshot_ids = {str(snapshot_id) for snapshot_id, _crawl_id in existing_snapshots}
+    existing_crawl_ids = {str(crawl_id) for _snapshot_id, crawl_id in existing_snapshots}
+    missing_snapshot_ids = sorted(str(snapshot_id) for snapshot_id in snapshot_ids - existing_snapshot_ids)
+    for snapshot_id in missing_snapshot_ids:
+        rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
 
-        requested_plugin_names = set(plugins_list) | requested_plugins_by_snapshot.get(str(snapshot.id), set())
-        for plugin_name in requested_plugin_names:
-            existing_result = snapshot.archiveresult_set.filter(plugin=plugin_name).order_by("-created_at").first()
-            if existing_result:
-                existing_result.reset_for_retry()
+    # Queue only the target plugin rows. Bulk updates keep large reindex runs
+    # from doing one SELECT+UPDATE per snapshot/plugin before hooks even start.
+    requested_pairs: set[tuple[str, str]] = set()
+    if plugins_list:
+        requested_pairs.update((snapshot_id, plugin_name) for snapshot_id, plugin_name in product(existing_snapshot_ids, plugins_list))
+    else:
+        requested_pairs.update(
+            (snapshot_id, plugin_name)
+            for snapshot_id, plugin_names in requested_plugins_by_snapshot.items()
+            if snapshot_id in existing_snapshot_ids
+            for plugin_name in plugin_names
+        )
+    plugins_by_name = discover_plugins(runtime="archivebox")
+    requested_rows: set[tuple[str, str, str]] = set()
+    for snapshot_id, plugin_name in requested_pairs:
+        plugin = plugins_by_name.get(plugin_name)
+        hooks = plugin.filter_hooks("Snapshot") if plugin is not None else []
+        if hooks:
+            requested_rows.update((snapshot_id, plugin_name, hook.name) for hook in hooks)
+        else:
+            requested_rows.add((snapshot_id, plugin_name, ""))
 
-        # Reset snapshot status to allow processing
-        if snapshot.status == Snapshot.StatusChoices.SEALED:
-            snapshot.status = Snapshot.StatusChoices.STARTED
-            snapshot.retry_at = timezone.now()
-            snapshot.save()
+    queued_rows: set[tuple[str, str, str]] = set()
+    if preserve_queued and requested_rows:
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+    rows_to_queue = requested_rows - queued_rows
 
-        processed_count += 1
+    reset_fields = {
+        "status": ArchiveResult.StatusChoices.QUEUED,
+        "output_str": "",
+        "output_json": None,
+        "output_files": {},
+        "output_size": 0,
+        "output_mimetypes": "",
+        "start_ts": None,
+        "end_ts": None,
+        "modified_at": timezone.now(),
+    }
+    if rows_to_queue and plugins_list:
+        rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    elif rows_to_queue and requested_plugins_by_snapshot:
+        snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    existing_rows = (
+        {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        if rows_to_queue
+        else set()
+    )
+    missing_rows = rows_to_queue - existing_rows
+    if missing_rows:
+        ArchiveResult.objects.bulk_create(
+            [
+                ArchiveResult(
+                    snapshot_id=snapshot_id,
+                    plugin=plugin_name,
+                    hook_name=hook_name,
+                    status=ArchiveResult.StatusChoices.QUEUED,
+                )
+                for snapshot_id, plugin_name, hook_name in sorted(missing_rows)
+            ],
+            batch_size=500,
+        )
+
+    processed_count = len(existing_snapshot_ids)
+    queue_at = timezone.now()
+    if existing_snapshot_ids:
+        if requested_rows:
+            # Targeted ArchiveResult retries use retry_at as the scheduling
+            # signal and keep sealed snapshots sealed so extractors are not
+            # re-run outside the explicitly queued plugin rows. Paused snapshots
+            # also keep status=paused here: `retry_at` only asks the orchestrator
+            # to process the queued plugin rows, and run_due_snapshot restores
+            # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
+            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
+            if preserve_queued and queued_rows:
+                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
+                    .filter(retry_at__gt=queue_at)
+                    .values_list("id", flat=True)
+                )
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list(
+                        "id",
+                        flat=True,
+                    )
+                )
+            for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
+                # Guard the read-time status so we never bump retry_at on a
+                # row that's been re-queued / started by a concurrent runner.
+                snapshot.safe_update(
+                    {"retry_at": queue_at, "modified_at": queue_at},
+                    refresh=False,
+                    extra_filter={"status": snapshot.status},
+                )
+        else:
+            # No plugin rows were requested, so this is a full snapshot retry.
+            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):
+                snapshot.safe_update(
+                    {
+                        "status": Snapshot.StatusChoices.QUEUED,
+                        "retry_at": queue_at,
+                        "current_step": 0,
+                        "modified_at": queue_at,
+                    },
+                    refresh=False,
+                    extra_filter={"status": snapshot.status},
+                )
+    if existing_crawl_ids and not requested_rows:
+        from archivebox.crawls.models import Crawl
+
+        for crawl in Crawl.objects.filter(id__in=existing_crawl_ids).only("id", "status", "retry_at", "modified_at"):
+            update_fields = {
+                "retry_at": queue_at,
+                "modified_at": queue_at,
+            }
+            if crawl.status != Crawl.StatusChoices.STARTED:
+                update_fields["status"] = Crawl.StatusChoices.QUEUED
+            crawl.safe_update(
+                update_fields,
+                refresh=False,
+                extra_filter={"status": crawl.status},
+            )
 
     if processed_count == 0:
         rprint("[red]No snapshots to process[/red]", file=sys.stderr)
         return 1
 
-    rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
+    if show_progress:
+        rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
 
     # Run orchestrator if --wait (default)
     if wait:
-        rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
+        if show_progress:
+            rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
         snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id in snapshot_ids:
-            try:
-                snapshot = Snapshot.objects.only("id", "crawl_id").get(id=snapshot_id)
-            except Snapshot.DoesNotExist:
-                continue
-            snapshot_ids_by_crawl[str(snapshot.crawl_id)].add(str(snapshot.id))
+        for snapshot_id, crawl_id in existing_snapshots:
+            snapshot_ids_by_crawl[str(crawl_id)].add(str(snapshot_id))
 
         for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items():
+            from archivebox.crawls.models import Crawl
+
+            crawl = Crawl.objects.get(id=crawl_id)
+            if not crawl.claim_processing_lock(lock_seconds=10):
+                rprint(
+                    f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]",
+                    file=sys.stderr,
+                )
+                return 1
             selected_plugins = (
                 plugins_list
                 or sorted(
@@ -222,6 +385,7 @@ def run_plugins(
                 crawl_id,
                 snapshot_ids=sorted(crawl_snapshot_ids),
                 selected_plugins=selected_plugins,
+                show_progress=show_progress,
             )
 
     if not emit_results:
@@ -255,16 +419,11 @@ def run_plugins(
 
 
 def is_archiveresult_id(value: str) -> bool:
-    """Check if value looks like an ArchiveResult UUID."""
-    import re
-
-    uuid_pattern = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
-    if not uuid_pattern.match(value):
-        return False
-    # Verify it's actually an ArchiveResult (not a Snapshot or other object)
+    """Check if value resolves to an ArchiveResult ID."""
     from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
 
-    return ArchiveResult.objects.filter(id=value).exists()
+    return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists()
 
 
 @click.command()

archivebox/cli/archivebox_help.py

@@ -2,6 +2,8 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox help"
 
+import ast
+import importlib.util
 import os
 from pathlib import Path
 
@@ -10,6 +12,29 @@ from rich import print
 from rich.panel import Panel
 
 
+def _command_doc(cmd: str, import_path: str) -> str:
+    def first_doc_line(docstring: str | None) -> str:
+        lines = (docstring or "").splitlines()
+        return lines[0] if lines else ""
+
+    modname, _ = import_path.rsplit(".", 1)
+    spec = importlib.util.find_spec(modname)
+    if spec is None or spec.origin is None:
+        return ""
+
+    try:
+        tree = ast.parse(Path(spec.origin).read_text())
+    except OSError:
+        return ""
+
+    for name in (cmd, "main"):
+        for node in tree.body:
+            if isinstance(node, ast.FunctionDef) and node.name == name:
+                return first_doc_line(ast.get_docstring(node))
+
+    return first_doc_line(ast.get_docstring(tree))
+
+
 def help() -> None:
     """Print the ArchiveBox help message and usage"""
 
@@ -22,16 +47,12 @@ def help() -> None:
     log_cli_command("help", [], None, ".")
 
     COMMANDS_HELP_TEXT = (
-        "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.meta_commands.keys()
-        )
+        "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.meta_commands.items())
+        + "\n\n    "
+        + "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.setup_commands.items())
         + "\n\n    "
         + "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.setup_commands.keys()
-        )
-        + "\n\n    "
-        + "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.archive_commands.keys()
+            f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.archive_commands.items()
         )
     )
 
@@ -68,8 +89,8 @@ def help() -> None:
     [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
 """)
 
-    config = get_config()
-    if os.access(config.ARCHIVE_DIR, os.R_OK) and config.ARCHIVE_DIR.is_dir():
+    get_config()
+    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
         pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
         EXAMPLE_USAGE = f"""
 [light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]

archivebox/cli/archivebox_init.py

@@ -25,10 +25,11 @@ def _display_data_path(path: Path, data_dir: Path) -> str:
 def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
     """Initialize a new ArchiveBox collection in the current directory"""
 
-    from archivebox.config import CONSTANTS, VERSION, DATA_DIR
+    from archivebox.config import CONSTANTS, VERSION
     from archivebox.config.common import get_config
     from archivebox.config.collection import write_config_file
     from archivebox.misc.db import apply_migrations
+    from archivebox.misc.checks import check_migrations
 
     config = get_config()
 
@@ -36,7 +37,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
     #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
 
-    is_empty = not len(set(os.listdir(DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
+    is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
     existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
     if is_empty and not existing_index:
         print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
@@ -64,21 +65,23 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     else:
         print("\n[green][+] Building archive folder structure...[/green]")
 
-    archive_path = _display_data_path(config.ARCHIVE_DIR, DATA_DIR)
-    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, DATA_DIR)
-    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, DATA_DIR)
+    archive_path = _display_data_path(CONSTANTS.ARCHIVE_DIR, CONSTANTS.DATA_DIR)
+    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, CONSTANTS.DATA_DIR)
+    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, CONSTANTS.DATA_DIR)
     print(f"    + {archive_path}, {sources_path}, {logs_path}...")
     Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
-    config.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
-    config.USERS_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.USERS_DIR.mkdir(parents=True, exist_ok=True)
     Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
+    for path in (Path(CONSTANTS.SOURCES_DIR), CONSTANTS.ARCHIVE_DIR, CONSTANTS.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
+        path.chmod(int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)
 
-    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, DATA_DIR)}...")
+    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, CONSTANTS.DATA_DIR)}...")
 
     # create the .archivebox_id file with a unique ID for this collection
     from archivebox.config.paths import _get_collection_id
 
-    _get_collection_id(DATA_DIR, force_create=True)
+    _get_collection_id(CONSTANTS.DATA_DIR, force_create=True)
 
     # create the ArchiveBox.conf file
     write_config_file({"SECRET_KEY": config.SECRET_KEY})
@@ -91,13 +94,22 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     from archivebox.config.django import setup_django
 
     setup_django()
+    previous_wants_init = os.environ.get("ARCHIVEBOX_WANTS_INIT")
+    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+    try:
+        check_migrations(blocking=True, auto_apply=False)
 
-    for migration_line in apply_migrations(DATA_DIR):
-        sys.stdout.write(f"    {migration_line}\n")
+        for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
+            sys.stdout.write(f"    {migration_line}\n")
+    finally:
+        if previous_wants_init is None:
+            os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
+        else:
+            os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
 
     assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
     print()
-    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, DATA_DIR)}")
+    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
 
     # from django.contrib.auth.models import User
     #     call_command("createsuperuser", interactive=True)
@@ -141,8 +153,8 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
 
     config = get_config()
     config.TMP_DIR.mkdir(parents=True, exist_ok=True)
-    config.LIB_DIR.mkdir(parents=True, exist_ok=True)
-    (config.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
+    config.ABXPKG_LIB_DIR.mkdir(parents=True, exist_ok=True)
+    (config.ABXPKG_LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
 
     working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
     if working_tmp_dir:
@@ -151,7 +163,6 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
     if working_lib_dir:
         working_lib_dir.mkdir(parents=True, exist_ok=True)
-        (working_lib_dir / "bin").mkdir(parents=True, exist_ok=True)
 
     if install:
         from archivebox.cli.archivebox_install import install as install_method
@@ -175,7 +186,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
 @click.command()
 @click.option("--force", "-f", is_flag=True, help="Ignore unrecognized files in current directory and initialize anyway")
 @click.option("--quick", "-q", is_flag=True, help="Run any updates or migrations without rechecking all snapshot dirs")
-@click.option("--install", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
+@click.option("--install", "--setup", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
 @docstring(init.__doc__)
 def main(**kwargs) -> None:
     init(**kwargs)

archivebox/cli/archivebox_install.py

@@ -3,6 +3,8 @@
 __package__ = "archivebox.cli"
 
 import os
+import shutil
+from pathlib import Path
 
 import rich_click as click
 from rich import print
@@ -10,6 +12,38 @@ from rich import print
 from archivebox.misc.util import docstring, enforce_types
 
 
+def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:
+    """Expose the shared abxpkg lib dir at DATA_DIR/lib after install."""
+    link_path = data_dir / "lib"
+    target_config_path = abxpkg_lib_dir.expanduser().absolute()
+    target_path = abxpkg_lib_dir.expanduser().resolve()
+    from archivebox.config.paths import get_machine_type
+
+    scoped_data_lib_path = (link_path / get_machine_type()).absolute()
+
+    if target_config_path == scoped_data_lib_path:
+        if link_path.is_symlink() or link_path.is_file():
+            link_path.unlink()
+        target_config_path.mkdir(parents=True, exist_ok=True)
+        return None
+
+    if link_path.resolve(strict=False) == target_path:
+        return None
+
+    target_path.mkdir(parents=True, exist_ok=True)
+
+    if link_path.is_symlink():
+        if link_path.resolve(strict=False) == target_path:
+            return link_path
+        link_path.unlink()
+    elif link_path.exists():
+        shutil.rmtree(link_path)
+
+    relative_target = os.path.relpath(target_path, start=link_path.parent)
+    link_path.symlink_to(relative_target, target_is_directory=True)
+    return link_path
+
+
 @enforce_types
 def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bool = False) -> None:
     """Detect and install ArchiveBox dependencies by running the abx-dl install flow
@@ -22,12 +56,16 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
     """
 
     from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
-    from archivebox.config.common import get_config
+    from archivebox.config import CONSTANTS
     from archivebox.misc.logging import stderr
     from archivebox.cli.archivebox_init import init
 
-    config = get_config()
-    archive_dir = config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
+
+    if dry_run:
+        print("[dim]Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow[/dim]")
+        return
+
     if not (os.access(archive_dir, os.R_OK) and archive_dir.is_dir()):
         init()  # must init full index because we need a db to store Binary entries in
 
@@ -47,10 +85,6 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
         print(f"    DATA_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].")
         print()
 
-    if dry_run:
-        print("[dim]Dry run - would run the abx-dl install flow[/dim]")
-        return
-
     # Set up Django
     from archivebox.config.django import setup_django
 
@@ -67,6 +101,11 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
 
     run_install(plugin_names=plugin_names or None)
 
+    from archivebox.config.common import get_config
+
+    config = get_config(include_machine=False)
+    ensure_data_dir_lib_symlink(CONSTANTS.DATA_DIR, config.ABXPKG_LIB_DIR)
+
     print()
 
     # Check for superuser

archivebox/cli/archivebox_list.py

@@ -7,50 +7,15 @@ import sys
 
 import rich_click as click
 
-from archivebox.cli.archivebox_snapshot import list_snapshots
+from archivebox.cli.archivebox_snapshot import list_snapshots, snapshot_filter_options, snapshot_output_options
 
 
 @click.command()
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def main(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-) -> None:
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def main(**kwargs) -> None:
     """List Snapshots."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_machine.py

@@ -27,7 +27,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================

archivebox/cli/archivebox_oneshot.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox oneshot"
+
+import subprocess
+from pathlib import Path
+
+import rich_click as click
+
+from archivebox.config import CONSTANTS
+
+
+@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
+@click.argument("args", nargs=-1)
+def main(args: tuple[str, ...] = ()) -> None:
+    """Download URLs using abx-dl"""
+    cwd = Path.cwd()
+    if any((path / CONSTANTS.SQL_INDEX_FILENAME).exists() for path in (cwd, *cwd.parents)):
+        raise click.ClickException(
+            "Refusing to run `archivebox oneshot` inside an ArchiveBox DATA_DIR. Use `archivebox add` here, or run oneshot from another directory.",
+        )
+    raise SystemExit(subprocess.run(["abx-dl", *args]).returncode)
+
+
+if __name__ == "__main__":
+    main()

archivebox/cli/archivebox_persona.py

@@ -31,17 +31,13 @@ import os
 import sys
 import shutil
 import platform
-import subprocess
-import tempfile
-import json
 from pathlib import Path
 from collections.abc import Iterable
-from collections import OrderedDict
 
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 from archivebox.personas import importers as persona_importers
 
 
@@ -214,136 +210,6 @@ CHROMIUM_BROWSERS = {"chrome", "chromium", "brave", "edge"}
 # Cookie Extraction via CDP
 # =============================================================================
 
-NETSCAPE_COOKIE_HEADER = [
-    "# Netscape HTTP Cookie File",
-    "# https://curl.se/docs/http-cookies.html",
-    "# This file was generated by ArchiveBox persona cookie extraction",
-    "#",
-    "# Format: domain\\tincludeSubdomains\\tpath\\tsecure\\texpiry\\tname\\tvalue",
-    "",
-]
-
-
-def _parse_netscape_cookies(path: Path) -> "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]":
-    cookies = OrderedDict()
-    if not path.exists():
-        return cookies
-
-    for line in path.read_text().splitlines():
-        if not line or line.startswith("#"):
-            continue
-        parts = line.split("\t")
-        if len(parts) < 7:
-            continue
-        domain, include_subdomains, cookie_path, secure, expiry, name, value = parts[:7]
-        key = (domain, cookie_path, name)
-        cookies[key] = (domain, include_subdomains, cookie_path, secure, expiry, name, value)
-    return cookies
-
-
-def _write_netscape_cookies(path: Path, cookies: "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]") -> None:
-    lines = list(NETSCAPE_COOKIE_HEADER)
-    for cookie in cookies.values():
-        lines.append("\t".join(cookie))
-    path.write_text("\n".join(lines) + "\n")
-
-
-def _merge_netscape_cookies(existing_file: Path, new_file: Path) -> None:
-    existing = _parse_netscape_cookies(existing_file)
-    new = _parse_netscape_cookies(new_file)
-    for key, cookie in new.items():
-        existing[key] = cookie
-    _write_netscape_cookies(existing_file, existing)
-
-
-def extract_cookies_via_cdp(
-    user_data_dir: Path,
-    output_file: Path,
-    profile_dir: str | None = None,
-    chrome_binary: str | None = None,
-) -> bool:
-    """
-    Launch Chrome with the given user data dir and extract cookies via CDP.
-
-    Returns True if successful, False otherwise.
-    """
-    from archivebox.config.common import get_config
-
-    # Find the cookie extraction script
-    chrome_plugin_dir = Path(__file__).parent.parent / "plugins" / "chrome"
-    extract_script = chrome_plugin_dir / "extract_cookies.js"
-
-    if not extract_script.exists():
-        rprint(f"[yellow]Cookie extraction script not found at {extract_script}[/yellow]", file=sys.stderr)
-        return False
-
-    # Get node modules dir
-    node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules"
-
-    # Set up environment
-    env = os.environ.copy()
-    env["NODE_MODULES_DIR"] = str(node_modules_dir)
-    env["CHROME_USER_DATA_DIR"] = str(user_data_dir)
-    env["CHROME_HEADLESS"] = "true"
-    if chrome_binary:
-        env["CHROME_BINARY"] = str(chrome_binary)
-    output_path = output_file
-    temp_output = None
-    temp_dir = None
-    if output_file.exists():
-        temp_dir = Path(tempfile.mkdtemp(prefix="ab_cookies_"))
-        temp_output = temp_dir / "cookies.txt"
-        output_path = temp_output
-    if profile_dir:
-        extra_arg = f"--profile-directory={profile_dir}"
-        existing_extra = env.get("CHROME_ARGS_EXTRA", "").strip()
-        args_list = []
-        if existing_extra:
-            if existing_extra.startswith("["):
-                try:
-                    parsed = json.loads(existing_extra)
-                    if isinstance(parsed, list):
-                        args_list.extend(str(x) for x in parsed)
-                except Exception:
-                    args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()])
-            else:
-                args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()])
-        args_list.append(extra_arg)
-        env["CHROME_ARGS_EXTRA"] = json.dumps(args_list)
-
-    env["COOKIES_OUTPUT_FILE"] = str(output_path)
-
-    try:
-        result = subprocess.run(
-            ["node", str(extract_script)],
-            env=env,
-            capture_output=True,
-            text=True,
-            timeout=60,
-        )
-
-        if result.returncode == 0:
-            if temp_output and temp_output.exists():
-                _merge_netscape_cookies(output_file, temp_output)
-            return True
-        else:
-            rprint(f"[yellow]Cookie extraction failed: {result.stderr}[/yellow]", file=sys.stderr)
-            return False
-
-    except subprocess.TimeoutExpired:
-        rprint("[yellow]Cookie extraction timed out[/yellow]", file=sys.stderr)
-        return False
-    except FileNotFoundError:
-        rprint("[yellow]Node.js not found. Cannot extract cookies.[/yellow]", file=sys.stderr)
-        return False
-    except Exception as e:
-        rprint(f"[yellow]Cookie extraction error: {e}[/yellow]", file=sys.stderr)
-        return False
-    finally:
-        if temp_dir and temp_dir.exists():
-            shutil.rmtree(temp_dir, ignore_errors=True)
-
-
 # =============================================================================
 # Validation Helpers
 # =============================================================================
@@ -506,7 +372,7 @@ def create_personas(
         if not is_tty:
             write_record(
                 {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "id": str(persona.id),
                     "name": persona.name,
                     "path": str(persona.path),
                     "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -558,7 +424,7 @@ def list_personas(
         else:
             write_record(
                 {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "id": str(persona.id),
                     "name": persona.name,
                     "path": str(persona.path),
                     "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -634,7 +500,7 @@ def update_personas(name: str | None = None) -> int:
             if not is_tty:
                 write_record(
                     {
-                        "id": str(persona.id) if hasattr(persona, "id") else None,
+                        "id": str(persona.id),
                         "name": persona.name,
                         "path": str(persona.path),
                     },

archivebox/cli/archivebox_pluginmap.py

@@ -16,8 +16,9 @@ EVENT_FLOW_DIAGRAM = """
 │  InstallEvent                                                               │
 │    └─ config.json > required_binaries                                       │
 │         └─ BinaryRequestEvent                                               │
-│              └─ on_BinaryRequest__*                                         │
+│              └─ abxpkg BinaryService builtin providers                      │
 │                   └─ BinaryEvent                                            │
+│                        └─ BinaryCacheService / project cache backend        │
 │                                                                             │
 │  CrawlEvent                                                                 │
 │    └─ CrawlSetupEvent                                                       │
@@ -56,13 +57,15 @@ def pluginmap(
     from rich.panel import Panel
     from rich import box
 
-    from archivebox.hooks import (
-        BUILTIN_PLUGINS_DIR,
-        USER_PLUGINS_DIR,
+    from archivebox.plugins.hooks import (
         discover_hooks,
         is_background_hook,
         normalize_hook_event_name,
     )
+    from archivebox.plugins.discovery import (
+        BUILTIN_PLUGINS_DIR,
+        USER_PLUGINS_DIR,
+    )
 
     console = Console()
     prnt = console.print
@@ -73,12 +76,14 @@ def pluginmap(
             "emits": ["BinaryRequestEvent", "BinaryEvent", "ProcessEvent"],
         },
         "BinaryRequestEvent": {
-            "description": "Provider phase. on_BinaryRequest hooks resolve or install requested binaries.",
+            "description": "Binary resolution phase. abxpkg BinaryService resolves or installs requested binaries using built-in providers.",
             "emits": ["BinaryEvent", "ProcessEvent"],
+            "direct_hooks": False,
         },
         "BinaryEvent": {
-            "description": "Resolved binary metadata event. Projected into the DB binary cache.",
+            "description": "Resolved binary metadata event. ArchiveBoxBinaryService projects it into the ArchiveBox DB binary cache.",
             "emits": [],
+            "direct_hooks": False,
         },
         "CrawlEvent": {
             "description": "Root crawl lifecycle event emitted by the runner.",
@@ -138,7 +143,8 @@ def pluginmap(
 
     for event_name, info in event_phases.items():
         hook_event = normalize_hook_event_name(event_name)
-        hooks = discover_hooks(event_name, filter_disabled=not show_disabled)
+        has_direct_hooks = info.get("direct_hooks", True)
+        hooks = discover_hooks(event_name, filter_disabled=not show_disabled) if has_direct_hooks else []
 
         hook_infos = []
         for hook_path in hooks:
@@ -192,8 +198,10 @@ def pluginmap(
         prnt(f"[dim]{info['description']}[/dim]")
         if info["emits"]:
             prnt(f"[dim]Emits: {', '.join(info['emits'])}[/dim]")
-        if not hook_infos:
+        if not hook_infos and has_direct_hooks:
             prnt(f"[dim]No direct on_{hook_event}__* scripts are currently defined for this event family.[/dim]")
+        elif not has_direct_hooks:
+            prnt("[dim]No direct plugin hook family. This event is handled by services.[/dim]")
         prnt()
 
     if not quiet:

archivebox/cli/archivebox_process.py

@@ -30,7 +30,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================
@@ -54,7 +54,7 @@ def list_processes(
 
     is_tty = sys.stdout.isatty()
 
-    queryset = Process.objects.all().select_related("binary", "machine").order_by("-start_ts")
+    queryset = Process.objects.all().select_related("binary", "machine").order_by("-started_at", "-created_at")
 
     # Apply filters
     filter_kwargs = {}

archivebox/cli/archivebox_remove.py

@@ -3,16 +3,16 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox remove"
 
-import shutil
+import time
 from pathlib import Path
 from collections.abc import Iterable
 
 import rich_click as click
 
+from django.db import OperationalError
 from django.db.models import QuerySet
 
-from archivebox.config import DATA_DIR
-from archivebox.config.common import get_config
+from archivebox.config import CONSTANTS
 from archivebox.config.django import setup_django
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.checks import check_data_folder
@@ -23,6 +23,7 @@ from archivebox.misc.logging_util import (
     log_removal_finished,
     TimedProgress,
 )
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
 
 @enforce_types
@@ -34,78 +35,123 @@ def remove(
     before: float | None = None,
     yes: bool = False,
     delete: bool = False,
-    out_dir: Path = DATA_DIR,
-) -> QuerySet:
+    out_dir: Path = CONSTANTS.DATA_DIR,
+    timeout: float | None = None,
+    **kwargs,
+) -> dict[str, object]:
     """Remove the specified URLs from the archive"""
 
     setup_django()
     check_data_folder()
+    timeout = float(timeout) if timeout is not None else None
 
-    from archivebox.cli.archivebox_search import get_snapshots
+    from archivebox.core.models import Snapshot
 
+    filter_kwargs = {
+        **kwargs,
+        "filter_patterns": filter_patterns,
+        "filter_type": filter_type,
+        "after": after,
+        "before": before,
+    }
     pattern_list = list(filter_patterns)
 
     log_list_started(pattern_list or None, filter_type)
     timer = TimedProgress(360, prefix="      ")
     try:
-        snapshots = get_snapshots(
-            snapshots=snapshots,
-            filter_patterns=pattern_list or None,
-            filter_type=filter_type,
-            after=after,
-            before=before,
-        )
+        if snapshots is None:
+            snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
+        # Freeze the target set up-front so a concurrent daemon writing new
+        # snapshots can't extend the deletion under us, and so the cursor isn't
+        # held open across the per-row deletes below.
+        snapshot_pks = list(snapshots.values_list("pk", flat=True))
     finally:
         timer.end()
 
-    if not snapshots.exists():
+    if not snapshot_pks:
         log_removal_finished(0, 0)
         raise SystemExit(1)
 
-    log_list_finished(snapshots)
-    log_removal_started(snapshots, yes=yes, delete=delete)
+    if not yes:
+        log_list_finished(snapshots)
+        log_removal_started(snapshots, yes=False)
 
-    timer = TimedProgress(360, prefix="      ")
-    try:
-        for snapshot in snapshots:
-            if delete:
-                shutil.rmtree(snapshot.output_dir, ignore_errors=True)
-                legacy_path = get_config().ARCHIVE_DIR / snapshot.timestamp
-                if legacy_path.is_symlink():
-                    legacy_path.unlink(missing_ok=True)
-    finally:
-        timer.end()
+    from archivebox.search.query import flush_search_index
 
-    to_remove = snapshots.count()
-
-    from archivebox.search import flush_search_index
-    from archivebox.core.models import Snapshot
+    started_at = time.monotonic()
+    deadline = started_at + timeout if timeout is not None else None
 
+    # Search-index flush touches a separate backend (FTS / sonic), not the
+    # main index.sqlite3 writer lock, so it's safe to do once up front.
     flush_search_index(snapshots=snapshots)
-    snapshots.delete()
-    all_snapshots = Snapshot.objects.all()
-    log_removal_finished(all_snapshots.count(), to_remove)
 
-    return all_snapshots
+    # Delete one snapshot at a time. Each ``.delete()`` is its own short
+    # Django-atomic block, so the writer lock is released between rows and
+    # an in-flight daemon transaction can interleave instead of deadlocking.
+    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
+    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
+    # rmtree doesn't hold the lock either.
+    #
+    # The SQLite retry wrapper in core/sqlite_backend/base.py re-raises lock
+    # errors when called inside an atomic block (because it can't safely
+    # release+reacquire a transaction), so we wrap each row's delete in our
+    # own retry loop at this outer (non-atomic) level. Each attempt is a
+    # fresh atomic; an exception cleanly rolls it back before we sleep.
+    retry_interval = 1.0
+    deleted_snapshot_pks = []
+    timed_out = False
+    timeout_error = ""
+    for index, pk in enumerate(snapshot_pks):
+        if deadline is not None and time.monotonic() >= deadline:
+            timed_out = True
+            timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining."
+            break
+        while True:
+            try:
+                deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
+                if deleted_count:
+                    deleted_snapshot_pks.append(pk)
+                break
+            except OperationalError as err:
+                if "database is locked" not in str(err):
+                    raise
+                remaining_time = deadline - time.monotonic() if deadline is not None else None
+                if remaining_time is not None and remaining_time <= 0:
+                    timed_out = True
+                    timeout_error = f"Remove timed out after {timeout:g}s while waiting for the database lock."
+                    break
+                time.sleep(min(retry_interval, remaining_time) if remaining_time is not None else retry_interval)
+        if timed_out:
+            break
+
+    all_snapshots = Snapshot.objects.all()
+    remaining_count = all_snapshots.count()
+    deleted_snapshot_id_set = set(deleted_snapshot_pks)
+    remaining_snapshot_pks = [snapshot_id for snapshot_id in snapshot_pks if snapshot_id not in deleted_snapshot_id_set]
+    log_removal_finished(remaining_count, len(deleted_snapshot_pks))
+
+    return {
+        "removed_count": len(deleted_snapshot_pks),
+        "removed_snapshot_ids": [str(snapshot_id) for snapshot_id in deleted_snapshot_pks],
+        "not_removed_count": len(remaining_snapshot_pks),
+        "not_removed_snapshot_ids": [str(snapshot_id) for snapshot_id in remaining_snapshot_pks],
+        "success": not timed_out,
+        "error": timeout_error,
+        "timeout": timeout,
+    }
 
 
 @click.command()
 @click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
-@click.option("--delete", is_flag=True, help="Delete the archived content and metadata folder in addition to removing from index")
-@click.option("--before", type=float, help="Remove only URLs bookmarked before timestamp")
-@click.option("--after", type=float, help="Remove only URLs bookmarked after timestamp")
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(("exact", "substring", "domain", "regex", "tag")),
-    default="exact",
-    help="Type of pattern matching to use when filtering URLs",
-)
-@click.argument("filter_patterns", nargs=-1)
+@click.option("--delete", is_flag=True, help="Compatibility no-op; archived content is deleted automatically")
+@click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots")
+@snapshot_filter_options(default_filter_type="exact")
 @docstring(remove.__doc__)
 def main(**kwargs):
     """Remove the specified URLs from the archive"""
-    remove(**kwargs)
+    result = remove(**kwargs)
+    if not result["success"]:
+        raise SystemExit(124)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_run.py

@@ -40,6 +40,9 @@ Examples:
 __package__ = "archivebox.cli"
 __command__ = "archivebox run"
 
+import asyncio
+import os
+import signal
 import sys
 from collections import defaultdict
 
@@ -47,6 +50,13 @@ import rich_click as click
 from rich import print as rprint
 
 
+RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
+
+
+def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
+    os._exit(128 + int(sig))
+
+
 def process_stdin_records() -> int:
     """
     Process JSONL records from stdin.
@@ -76,6 +86,7 @@ def process_stdin_records() -> int:
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot, ArchiveResult
     from archivebox.crawls.models import Crawl
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Binary
     from archivebox.services.runner import run_binary, run_crawl
 
@@ -111,10 +122,10 @@ def process_stdin_records() -> int:
                     crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if crawl:
-                    crawl.retry_at = timezone.now()
-                    if crawl.status not in [Crawl.StatusChoices.SEALED]:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save()
+                    crawl.update_and_requeue(
+                        status=Crawl.StatusChoices.QUEUED,
+                        retry_at=timezone.now(),
+                    )
                     full_crawl_ids.add(str(crawl.id))
                     run_all_plugins_for_crawl.add(str(crawl.id))
                     output_records.append(crawl.to_json())
@@ -132,15 +143,7 @@ def process_stdin_records() -> int:
                     snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if snapshot:
-                    snapshot.retry_at = timezone.now()
-                    if snapshot.status not in [Snapshot.StatusChoices.SEALED]:
-                        snapshot.status = Snapshot.StatusChoices.QUEUED
-                    snapshot.save()
-                    crawl = snapshot.crawl
-                    crawl.retry_at = timezone.now()
-                    if crawl.status != Crawl.StatusChoices.STARTED:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save(update_fields=["status", "retry_at", "modified_at"])
+                    snapshot.queue_for_extraction()
                     crawl_id = str(snapshot.crawl_id)
                     snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
                     run_all_plugins_for_crawl.add(crawl_id)
@@ -177,15 +180,7 @@ def process_stdin_records() -> int:
                         snapshot = None
 
                 if snapshot:
-                    snapshot.retry_at = timezone.now()
-                    if snapshot.status != Snapshot.StatusChoices.STARTED:
-                        snapshot.status = Snapshot.StatusChoices.QUEUED
-                    snapshot.save(update_fields=["status", "retry_at", "modified_at"])
-                    crawl = snapshot.crawl
-                    crawl.retry_at = timezone.now()
-                    if crawl.status != Crawl.StatusChoices.STARTED:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save(update_fields=["status", "retry_at", "modified_at"])
+                    snapshot.queue_for_extraction()
                     crawl_id = str(snapshot.crawl_id)
                     snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
                     if plugin_name:
@@ -236,15 +231,29 @@ def process_stdin_records() -> int:
     targeted_crawl_ids = full_crawl_ids | set(snapshot_ids_by_crawl)
     if targeted_crawl_ids:
         for crawl_id in sorted(targeted_crawl_ids):
-            run_crawl(
-                crawl_id,
-                snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
-                selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
-            )
+            try:
+                crawl = Crawl.objects.get(id=crawl_id)
+            except Crawl.DoesNotExist:
+                continue
+            if not crawl.claim_processing_lock(lock_seconds=10):
+                rprint(f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]", file=sys.stderr)
+                return 1
+            with foreground_shutdown_signals(), foreground_parent_watchdog():
+                run_crawl(
+                    crawl_id,
+                    snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
+                    selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
+                    selected_plugins_are_explicit=False,
+                )
     return 0
 
 
-def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int:
+def run_runner(
+    daemon: bool = False,
+    crawl_id: str | None = None,
+    maintenance_only: bool = False,
+    maintenance_batch_size: int | None = None,
+) -> int:
     """
     Run the background runner loop.
 
@@ -253,35 +262,85 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int:
 
     Returns exit code (0 = success, 1 = error).
     """
-    from django.utils import timezone
+    from archivebox.config import CONSTANTS
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Machine, Process
-    from archivebox.services.runner import recover_orphaned_crawls, recover_orphaned_snapshots, run_pending_crawls
+    from archivebox.core.takeover_util import enter_single_runner_gate, standby_until_foreground_runner_needed
+    from archivebox.core.recovery_util import recover_orchestrator_state
+    from archivebox.services.runner import run_pending_crawls
 
-    Process.cleanup_stale_running()
-    Process.cleanup_orphaned_workers()
-    Process.cleanup_orphaned_chrome()
-    recover_orphaned_snapshots()
-    recover_orphaned_crawls()
     Machine.current()
     current = Process.current()
-    if current.process_type != Process.TypeChoices.ORCHESTRATOR:
-        current.process_type = Process.TypeChoices.ORCHESTRATOR
-        current.save(update_fields=["process_type", "modified_at"])
+    root_command = current.root
+    if daemon and root_command.process_type in (
+        Process.TypeChoices.SERVER,
+        Process.TypeChoices.ADD,
+        Process.TypeChoices.UPDATE,
+    ):
+        # Server-owned daemon runners are persistent supervisor workers, but
+        # foreground add/update commands are allowed to borrow runner/sonic
+        # leadership without taking down Daphne. Waiting here keeps the worker
+        # on the normal runner path while preventing a server restart loop from
+        # immediately stealing the single-runner gate back from the newer CLI.
+        standby_until_foreground_runner_needed(root_command, data_dir=CONSTANTS.DATA_DIR)
+    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
+        current.mark_exited()
+        return 0
 
+    recover_orchestrator_state(include_chrome=crawl_id is None, crawl_id=crawl_id)
+    if crawl_id:
+        from django.utils import timezone
+        from archivebox.crawls.models import Crawl
+
+        crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
+        now = timezone.now()
+        # Only re-lease when the row is unscheduled (retry_at IS NULL) or its
+        # existing lease has already expired. A future retry_at means another
+        # worker is already scheduled — don't clobber.
+        if crawl is not None and (crawl.retry_at is None or crawl.retry_at <= now):
+            # extra_filter pins the read-time retry_at so a concurrent worker
+            # that grabbed the lease between our SELECT and UPDATE wins.
+            crawl.safe_update(
+                {"retry_at": now},
+                refresh=False,
+                extra_filter={"retry_at": crawl.retry_at},
+            )
+    # Only a foreground `archivebox add` gets the interactive "abort current
+    # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
+    # orchestrators should shut down immediately and cleanly on the first signal.
+    interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
+    if daemon:
+        os.environ[RUNNER_DAEMON_ENV] = "1"
     try:
-        run_pending_crawls(daemon=daemon, crawl_id=crawl_id)
+        with (
+            foreground_shutdown_signals(
+                on_signal=_exit_daemon_runner_on_signal if daemon else None,
+                raise_on_first_signal=not daemon,
+            ),
+            foreground_parent_watchdog(enabled=not daemon),
+        ):
+            run_pending_crawls(
+                daemon=daemon,
+                crawl_id=crawl_id,
+                maintenance_only=maintenance_only,
+                interactive_interrupts=interactive_interrupts,
+                **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
+            )
         return 0
     except KeyboardInterrupt:
         return 0
+    except asyncio.CancelledError as e:
+        if daemon:
+            rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+            return 1
+        return 0
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
         return 1
     finally:
         current.refresh_from_db()
         if current.status != Process.StatusChoices.EXITED:
-            current.status = Process.StatusChoices.EXITED
-            current.ended_at = current.ended_at or timezone.now()
-            current.save(update_fields=["status", "ended_at", "modified_at"])
+            current.mark_exited()
 
 
 @click.command()
@@ -289,7 +348,24 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int:
 @click.option("--crawl-id", help="Run the crawl runner for a specific crawl only")
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
-def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str):
+@click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
+@click.option(
+    "--maintenance-batch-size",
+    type=int,
+    default=None,
+    hidden=True,
+    help="Limit queued maintenance snapshots claimed per scheduler tick",
+)
+@click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
+def main(
+    daemon: bool,
+    crawl_id: str,
+    snapshot_id: str,
+    binary_id: str,
+    maintenance_only: bool,
+    maintenance_batch_size: int | None,
+    no_stdin: bool,
+):
     """
     Process queued work.
 
@@ -300,45 +376,96 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str):
     - --snapshot-id: Run one snapshot through its crawl only
     - --binary-id: Run one queued binary install directly on the bus
     """
-    if snapshot_id:
-        sys.exit(run_snapshot_worker(snapshot_id))
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
 
-    if binary_id:
+    if daemon and not snapshot_id and not binary_id and not crawl_id:
         try:
-            from archivebox.services.runner import run_binary
-
-            run_binary(binary_id)
-            sys.exit(0)
+            os.environ[RUNNER_DAEMON_ENV] = "1"
+            with (
+                foreground_shutdown_signals(
+                    on_signal=_exit_daemon_runner_on_signal,
+                    raise_on_first_signal=False,
+                ),
+                foreground_parent_watchdog(enabled=False),
+            ):
+                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
         except KeyboardInterrupt:
             sys.exit(0)
-        except Exception as e:
-            rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-            import traceback
 
-            traceback.print_exc()
-            sys.exit(1)
+    with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
+        if snapshot_id:
+            sys.exit(run_snapshot_worker(snapshot_id))
 
-    if crawl_id:
-        sys.exit(run_runner(daemon=False, crawl_id=crawl_id))
+        if binary_id:
+            try:
+                from archivebox.services.runner import run_binary
 
-    if daemon:
-        sys.exit(run_runner(daemon=True))
+                run_binary(binary_id)
+                sys.exit(0)
+            except KeyboardInterrupt:
+                sys.exit(0)
+            except Exception as e:
+                rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+                import traceback
 
-    if not sys.stdin.isatty():
-        sys.exit(process_stdin_records())
-    else:
-        sys.exit(run_runner(daemon=daemon))
+                traceback.print_exc()
+                sys.exit(1)
+
+        if crawl_id:
+            sys.exit(
+                run_runner(
+                    daemon=False,
+                    crawl_id=crawl_id,
+                    maintenance_only=maintenance_only,
+                    maintenance_batch_size=maintenance_batch_size,
+                ),
+            )
+
+        if maintenance_only:
+            sys.exit(run_runner(daemon=daemon, maintenance_only=True, maintenance_batch_size=maintenance_batch_size))
+
+        if not no_stdin and not sys.stdin.isatty():
+            sys.exit(process_stdin_records())
+        else:
+            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
 
 
 def run_snapshot_worker(snapshot_id: str) -> int:
+    from archivebox.config import CONSTANTS
+    from archivebox.core.takeover_util import enter_single_runner_gate
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    from archivebox.machine.models import Process
     from archivebox.core.models import Snapshot
-    from archivebox.services.runner import run_crawl
+    from archivebox.services.runner import run_due_snapshot
+    from django.utils import timezone
 
+    current = Process.current()
+    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
+        current.mark_exited()
+        return 0
+
+    snapshot = None
     try:
-        snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)])
+        with foreground_shutdown_signals(), foreground_parent_watchdog():
+            for _ in range(10):
+                snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+                if snapshot.retry_at is None:
+                    snapshot.update_and_requeue(retry_at=timezone.now())
+                elif snapshot.retry_at > timezone.now():
+                    break
+                if not run_due_snapshot(snapshot, lock_seconds=60):
+                    break
         return 0
     except KeyboardInterrupt:
+        try:
+            if snapshot is not None:
+                snapshot.refresh_from_db()
+            else:
+                snapshot = Snapshot.objects.filter(id=snapshot_id).first()
+            if snapshot is not None and snapshot.status != Snapshot.StatusChoices.SEALED:
+                snapshot.update_and_requeue(retry_at=timezone.now())
+        except Exception:
+            pass
         return 0
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
@@ -346,6 +473,10 @@ def run_snapshot_worker(snapshot_id: str) -> int:
 
         traceback.print_exc()
         return 1
+    finally:
+        current.refresh_from_db()
+        if current.status != Process.StatusChoices.EXITED:
+            current.mark_exited()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_schedule.py

@@ -6,7 +6,6 @@ import rich_click as click
 from rich import print
 
 from archivebox.misc.util import enforce_types, docstring
-from archivebox.config.common import get_config
 
 
 @enforce_types
@@ -20,9 +19,8 @@ def schedule(
     every: str | None = None,
     tag: str = "",
     depth: int | str = 0,
-    overwrite: bool = False,
-    update: bool | None = None,
     import_path: str | None = None,
+    config: dict[str, object] | None = None,
 ):
     """Manage database-backed scheduled crawls processed by the crawl runner."""
 
@@ -30,12 +28,10 @@ def schedule(
 
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl, CrawlSchedule
-    from archivebox.crawls.schedule_utils import validate_schedule
+    from archivebox.crawls.schedule_util import validate_schedule
     from archivebox.services.runner import run_pending_crawls
 
-    if update is None:
-        update = not get_config().ONLY_NEW
-
+    config_overrides = dict(config or {})
     depth = int(depth)
     result: dict[str, object] = {
         "created_schedule_ids": [],
@@ -79,10 +75,12 @@ def schedule(
             status=Crawl.StatusChoices.SEALED,
             retry_at=None,
             config={
-                "ONLY_NEW": not update,
-                "OVERWRITE": overwrite,
                 "DEPTH": 0 if is_update_schedule else depth,
                 "SCHEDULE_KIND": "update" if is_update_schedule else "crawl",
+                # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) win over the
+                # template defaults. Anything left unset falls through to the
+                # standard config stack at crawl-resolution time.
+                **config_overrides,
             },
         )
         crawl_schedule = CrawlSchedule.objects.create(
@@ -163,8 +161,13 @@ def schedule(
     default="0",
     help="Recursively archive linked pages up to N hops away",
 )
-@click.option("--overwrite", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
-@click.option("--update", is_flag=True, help="Retry previously failed/skipped URLs when scheduled crawls run")
+@click.option(
+    "--only-new/--no-only-new",
+    "only_new",
+    default=None,
+    help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
+    "Pass --no-only-new to force re-archive on each scheduled run.",
+)
 @click.option("--clear", is_flag=True, help="Disable all currently enabled schedules")
 @click.option("--show", is_flag=True, help="Print all currently enabled schedules")
 @click.option("--foreground", "-f", is_flag=True, help="Run the global crawl runner in the foreground (no crontab required)")
@@ -173,6 +176,9 @@ def schedule(
 @docstring(schedule.__doc__)
 def main(**kwargs):
     """Manage database-backed scheduled crawls processed by the crawl runner."""
+    only_new = kwargs.pop("only_new", None)
+    if only_new is not None:
+        kwargs["config"] = {"ONLY_NEW": bool(only_new)}
     schedule(**kwargs)
 
 

archivebox/cli/archivebox_search.py

@@ -3,255 +3,7 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox search"
 
-import sys
-from pathlib import Path
-from typing import TYPE_CHECKING
-from collections.abc import Callable
-
-import rich_click as click
-
-from django.db.models import Q, QuerySet
-
-from archivebox.config import DATA_DIR
-from archivebox.config.common import get_config
-from archivebox.misc.logging import stderr
-from archivebox.misc.util import enforce_types, docstring
-
-if TYPE_CHECKING:
-    from archivebox.core.models import Snapshot
-
-# Filter types for URL matching
-LINK_FILTERS: dict[str, Callable[[str], Q]] = {
-    "exact": lambda pattern: Q(url=pattern),
-    "substring": lambda pattern: Q(url__icontains=pattern),
-    "regex": lambda pattern: Q(url__iregex=pattern),
-    "domain": lambda pattern: (
-        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
-    ),
-    "tag": lambda pattern: Q(tags__name=pattern),
-    "timestamp": lambda pattern: Q(timestamp=pattern),
-}
-
-STATUS_CHOICES = ["indexed", "archived", "unarchived"]
-
-
-def _apply_pattern_filters(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    filter_patterns: list[str],
-    filter_type: str,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    filter_builder = LINK_FILTERS.get(filter_type)
-    if filter_builder is None:
-        stderr()
-        stderr(f"[X] Got invalid pattern for --filter-type={filter_type}", color="red")
-        raise SystemExit(2)
-
-    query = Q()
-    for pattern in filter_patterns:
-        query |= filter_builder(pattern)
-    return snapshots.filter(query)
-
-
-def _snapshots_to_json(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from archivebox.config import VERSION
-    from archivebox.misc.util import to_json
-
-    config = get_config()
-    main_index_header = (
-        {
-            "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
-            "schema": "archivebox.index.json",
-            "copyright_info": config.FOOTER_INFO,
-            "meta": {
-                "project": "ArchiveBox",
-                "version": VERSION,
-                "git_sha": VERSION,
-                "website": "https://ArchiveBox.io",
-                "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
-                "source": "https://github.com/ArchiveBox/ArchiveBox",
-                "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
-                "dependencies": {},
-            },
-        }
-        if with_headers
-        else {}
-    )
-
-    snapshot_dicts = [snapshot.to_dict(extended=True) for snapshot in snapshots.iterator(chunk_size=500)]
-    output: dict[str, object] | list[dict[str, object]]
-    if with_headers:
-        output = {
-            **main_index_header,
-            "num_links": len(snapshot_dicts),
-            "updated": datetime.now(tz.utc),
-            "last_run_cmd": sys.argv,
-            "links": snapshot_dicts,
-        }
-    else:
-        output = snapshot_dicts
-
-    return to_json(output, indent=4, sort_keys=True)
-
-
-def _snapshots_to_csv(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    cols: list[str],
-    with_headers: bool,
-) -> str:
-    header = ",".join(cols) if with_headers else ""
-    rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.iterator(chunk_size=500)]
-    return "\n".join((header, *rows))
-
-
-def _snapshots_to_html(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from django.template.loader import render_to_string
-
-    from archivebox.config import VERSION
-    from archivebox.config.version import get_COMMIT_HASH
-
-    config = get_config()
-    template = "static_index.html" if with_headers else "minimal_index.html"
-    snapshot_list = list(snapshots.iterator(chunk_size=500))
-
-    return render_to_string(
-        template,
-        {
-            "version": VERSION,
-            "git_sha": get_COMMIT_HASH() or VERSION,
-            "num_links": str(len(snapshot_list)),
-            "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"),
-            "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"),
-            "links": snapshot_list,
-            "FOOTER_INFO": config.FOOTER_INFO,
-        },
-    )
-
-
-def get_snapshots(
-    snapshots: QuerySet["Snapshot", "Snapshot"] | None = None,
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    after: float | None = None,
-    before: float | None = None,
-    out_dir: Path = DATA_DIR,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    """Filter and return Snapshots matching the given criteria."""
-    from archivebox.core.models import Snapshot
-
-    if snapshots is not None:
-        result = snapshots
-    else:
-        result = Snapshot.objects.all()
-
-    if after is not None:
-        result = result.filter(timestamp__gte=after)
-    if before is not None:
-        result = result.filter(timestamp__lt=before)
-    if filter_patterns:
-        result = _apply_pattern_filters(result, filter_patterns, filter_type)
-
-    # Prefetch crawl relationship to avoid N+1 queries when accessing output_dir
-    result = result.select_related("crawl", "crawl__created_by")
-
-    if not result.exists():
-        stderr("[!] No Snapshots matched your filters:", filter_patterns, f"({filter_type})", color="lightyellow")
-
-    return result
-
-
-@enforce_types
-def search(
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    status: str = "indexed",
-    before: float | None = None,
-    after: float | None = None,
-    sort: str | None = None,
-    json: bool = False,
-    html: bool = False,
-    csv: str | None = None,
-    with_headers: bool = False,
-):
-    """List, filter, and export information about archive entries"""
-
-    if with_headers and not (json or html or csv):
-        stderr("[X] --with-headers requires --json, --html or --csv\n", color="red")
-        raise SystemExit(2)
-
-    # Query DB directly - no filesystem scanning
-    snapshots = get_snapshots(
-        filter_patterns=list(filter_patterns) if filter_patterns else None,
-        filter_type=filter_type,
-        before=before,
-        after=after,
-    )
-
-    # Apply status filter
-    if status == "archived":
-        snapshots = snapshots.filter(downloaded_at__isnull=False)
-    elif status == "unarchived":
-        snapshots = snapshots.filter(downloaded_at__isnull=True)
-    # 'indexed' = all snapshots (no filter)
-
-    if sort:
-        snapshots = snapshots.order_by(sort)
-
-    # Export to requested format
-    if json:
-        output = _snapshots_to_json(snapshots, with_headers=with_headers)
-    elif html:
-        output = _snapshots_to_html(snapshots, with_headers=with_headers)
-    elif csv:
-        output = _snapshots_to_csv(snapshots, cols=csv.split(","), with_headers=with_headers)
-    else:
-        from archivebox.misc.logging_util import printable_folders
-
-        # Convert to dict for printable_folders
-        folders: dict[str, Snapshot | None] = {str(snapshot.output_dir): snapshot for snapshot in snapshots}
-        output = printable_folders(folders, with_headers)
-
-    # Structured exports must be written directly to stdout.
-    # rich.print() reflows long lines to console width, which corrupts JSON/CSV/HTML output.
-    sys.stdout.write(output)
-    if not output.endswith("\n"):
-        sys.stdout.write("\n")
-    return output
-
-
-@click.command()
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(["search", *LINK_FILTERS.keys()]),
-    default="substring",
-    help="Pattern matching type for filtering URLs",
-)
-@click.option("--status", "-s", type=click.Choice(STATUS_CHOICES), default="indexed", help="List snapshots with the given status")
-@click.option("--before", "-b", type=float, help="List snapshots bookmarked before the given UNIX timestamp")
-@click.option("--after", "-a", type=float, help="List snapshots bookmarked after the given UNIX timestamp")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--json", "-J", is_flag=True, help="Print output in JSON format")
-@click.option("--html", "-M", is_flag=True, help="Print output in HTML format (suitable for viewing statically without a server)")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: created_at,url,title")
-@click.option("--with-headers", "-H", is_flag=True, help="Include extra CSV/HTML headers in the output")
-@click.help_option("--help", "-h")
-@click.argument("filter_patterns", nargs=-1)
-@docstring(search.__doc__)
-def main(**kwargs):
-    return search(**kwargs)
+from archivebox.cli.archivebox_list import main
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_server.py

@@ -4,126 +4,253 @@ __package__ = "archivebox.cli"
 
 import sys
 import os
+import socket
+import subprocess
+import time
 from collections.abc import Iterable
 
 import rich_click as click
 from rich import print
 
+from archivebox.config import CONSTANTS
 from archivebox.misc.util import docstring, enforce_types
-from archivebox.config.common import get_config
 
 
-def stop_existing_background_runner(*, machine, process_model, supervisor=None, stop_worker_fn=None, log=print) -> int:
-    """Stop any existing orchestrator process so the server can take ownership."""
-    process_model.cleanup_stale_running(machine=machine)
-    process_model.cleanup_orphaned_workers()
+import re as _re
 
-    running_runners = list(
-        process_model.objects.filter(
-            machine=machine,
-            status=process_model.StatusChoices.RUNNING,
-            process_type=process_model.TypeChoices.ORCHESTRATOR,
-        ).order_by("created_at"),
-    )
-
-    if not running_runners:
-        return 0
-
-    log("[yellow][*] Stopping existing ArchiveBox background runner...[/yellow]")
-
-    if supervisor is not None and stop_worker_fn is not None:
-        for worker_name in ("worker_runner", "worker_runner_watch"):
-            try:
-                stop_worker_fn(supervisor, worker_name)
-            except Exception:
-                pass
-
-    for proc in running_runners:
-        try:
-            proc.kill_tree(graceful_timeout=2.0)
-        except Exception:
-            try:
-                proc.terminate(graceful_timeout=2.0)
-            except Exception:
-                pass
-
-    process_model.cleanup_stale_running(machine=machine)
-    return len(running_runners)
+_IPV4_RE = _re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
+_IPV6_CHARS_RE = _re.compile(r"^[0-9a-fA-F:.]+$")
+_LOCAL_BIND_HOSTS = frozenset({"0.0.0.0", "::", "::0", "127.0.0.1", "::1"})
 
 
-def _read_supervisor_worker_command(worker_name: str) -> str:
-    from archivebox.workers.supervisord_util import WORKERS_DIR_NAME, get_sock_file
-
-    worker_conf = get_sock_file().parent / WORKERS_DIR_NAME / f"{worker_name}.conf"
-    if not worker_conf.exists():
-        return ""
-
-    for line in worker_conf.read_text().splitlines():
-        if line.startswith("command="):
-            return line.removeprefix("command=").strip()
-    return ""
+def _is_ipv4_literal(host: str) -> bool:
+    return bool(_IPV4_RE.match(host))
 
 
-def _worker_command_matches_bind(command: str, host: str, port: str) -> bool:
-    if not command:
+def _is_ipv6_literal(host: str) -> bool:
+    # Bracketed (e.g. ``[2001:db8::1]``) or bare form. Require at least two
+    # colons so we don't catch random strings with one ``:``.
+    stripped = host.strip("[]")
+    return stripped.count(":") >= 2 and bool(_IPV6_CHARS_RE.match(stripped))
+
+
+def _bind_host_looks_like_ip(host: str) -> bool:
+    if not host or host in _LOCAL_BIND_HOSTS:
         return False
-    return f"{host}:{port}" in command or (f"--bind={host}" in command and f"--port={port}" in command)
+    return _is_ipv4_literal(host) or _is_ipv6_literal(host)
 
 
-def stop_existing_server_workers(*, supervisor, stop_worker_fn, host: str, port: str, log=print) -> int:
-    """Stop existing ArchiveBox web workers if they already own the requested bind."""
-    stopped = 0
+def _split_bind_spec(spec: str) -> tuple[str, str]:
+    """Split a ``host:port`` / ``host`` / ``port`` spec into ``(host, port)``.
+
+    The empty strings stand in for "not provided"; the caller fills in
+    defaults. Bracketed IPv6 literals like ``[::1]:8000`` are handled.
+    """
+    spec = (spec or "").strip()
+    if not spec:
+        return "", ""
+    if spec.startswith("["):
+        # Bracketed IPv6: ``[::1]`` or ``[::1]:8000``
+        end = spec.find("]")
+        if end == -1:
+            return spec, ""  # malformed; let validator reject it
+        host = spec[: end + 1]
+        rest = spec[end + 1 :]
+        if rest.startswith(":"):
+            return host, rest[1:]
+        return host, ""
+    if ":" in spec:
+        host, _, port = spec.rpartition(":")
+        return host, port
+    # Bare token: digits = port, anything else = host
+    if spec.isdigit():
+        return "", spec
+    return spec, ""
+
+
+def _parse_and_validate_bind_spec(spec: str) -> tuple[str, str]:
+    """Resolve a CLI/config bind spec to ``(host, port)`` or hard-error.
+
+    Accepts only IP literals (v4 or v6) or the special string ``localhost``
+    (normalized to ``127.0.0.1``). Bare hostnames are rejected because the
+    bind address feeds Daphne, which has to listen on a numeric address;
+    public hostnames belong in ``BASE_URL`` instead. Empty values fall back
+    to ``127.0.0.1`` / ``8000``.
+    """
+    raw_host, raw_port = _split_bind_spec(spec)
+    host = raw_host.strip()
+    port = (raw_port or "").strip() or "8000"
+
+    if host == "" or host.lower() == "localhost":
+        host = "127.0.0.1"
+    elif _is_ipv4_literal(host) or _is_ipv6_literal(host):
+        pass
+    else:
+        print(
+            f"[red][X] Invalid BIND_ADDR host {host!r}: must be an IP literal or 'localhost'.[/red]",
+        )
+        print(
+            "[red]    Hostnames like archive.example.com are not valid bind addresses — Daphne[/red]",
+        )
+        print(
+            "[red]    listens on numeric addresses only. Bind to 0.0.0.0 and set BASE_URL instead:[/red]",
+        )
+        print(
+            f"[red]      BASE_URL=https://{host} archivebox server 0.0.0.0:{port}[/red]",
+        )
+        sys.exit(1)
+
+    try:
+        port_int = int(port)
+    except ValueError:
+        print(f"[red][X] Invalid BIND_ADDR port {port!r}: must be an integer 1-65535.[/red]")
+        sys.exit(1)
+    if not (0 < port_int < 65536):
+        print(f"[red][X] Invalid BIND_ADDR port {port_int}: must be 1-65535.[/red]")
+        sys.exit(1)
+
+    return host, port
+
+
+def _print_server_startup_warnings(config, host: str, port: str) -> None:
+    """Print startup-time security / routing warnings for the server command.
+
+    Runs only from ``archivebox server`` so other entry points (manage shell,
+    plugin lookups, etc.) don't repeat this banner on every config load.
+    """
+    if config.IS_LOWER_SECURITY_MODE:
+        print(
+            f"[yellow][!] WARNING: ArchiveBox is running with SERVER_SECURITY_MODE={config.SERVER_SECURITY_MODE}[/yellow]",
+        )
+        print("[yellow]    Archived pages may share an origin with privileged app routes in this mode.[/yellow]")
+        print("[yellow]    To switch to the safer isolated setup:[/yellow]")
+        print("[yellow]      1. Set SERVER_SECURITY_MODE=safe-subdomains-fullreplay[/yellow]")
+        print("[yellow]      2. Point *.archivebox.localhost (or your chosen base domain) at this server[/yellow]")
+        print(
+            "[yellow]      3. Configure wildcard DNS/TLS or your reverse proxy so admin., web., api., and snapshot subdomains resolve[/yellow]",
+        )
+        print()
+
+    # ``config.BASE_URL`` is the merged value (env > Machine.config > file >
+    # default), which is what the running server will actually use. Earlier we
+    # gated the "BASE_URL not set" warning on ``os.environ["BASE_URL"]`` alone,
+    # which fired noisily when the user pinned BASE_URL via Machine.config /
+    # ArchiveBox.conf instead of via env.
+    base_url = (config.BASE_URL or "").strip()
+    if base_url:
+        # BASE_URL is pinned. The only thing left to surface is a port
+        # mismatch — bind port ≠ BASE_URL's explicit port usually means the
+        # operator started the server with the wrong ``archivebox server PORT``
+        # argument (or forgot to update one side after moving the listener).
+        # A reverse-proxy setup typically omits the port in BASE_URL
+        # (``https://archive.example.com``), so we only warn when BASE_URL
+        # carries an explicit port — otherwise we'd nag every proxy deployment.
+        from urllib.parse import urlparse
 
-    for worker_name in ("worker_runserver", "worker_daphne"):
         try:
-            proc = supervisor.getProcessInfo(worker_name) if supervisor else None
-        except Exception:
-            proc = None
-        if not isinstance(proc, dict) or proc.get("statename") != "RUNNING":
-            continue
+            base_port = urlparse(base_url).port
+        except (ValueError, TypeError):
+            base_port = None
+        if base_port is not None and str(base_port) != str(port):
+            print(
+                f"[yellow][!] BASE_URL ({base_url}) port {base_port} does not match the port the server is running on ({port}). "
+                "Make sure this is intentional![/yellow]",
+            )
+            print()
+        return
 
-        command = _read_supervisor_worker_command(worker_name)
-        if not _worker_command_matches_bind(command, host, port):
-            continue
+    # If the user is upgrading from 0.7.3 and already had
+    # CSRF_TRUSTED_ORIGINS set, get_base_url() will silently use that as the
+    # implicit BASE_URL. Surface what we picked so the user knows where their
+    # links / redirects are going — and tell them how to make it explicit.
+    from archivebox.core.routes_util import derive_base_url_from_csrf
 
-        if stopped == 0:
-            log("[yellow][*] Taking over existing ArchiveBox web server on same port...[/yellow]")
-        stop_worker_fn(supervisor, worker_name)
-        stopped += 1
+    csrf_derived = derive_base_url_from_csrf(config)
+    if csrf_derived:
+        print(
+            f"[yellow][!] BASE_URL is not set; auto-derived [bold]{csrf_derived}[/bold] from a single CSRF_TRUSTED_ORIGINS entry.[/yellow]",
+        )
+        print(
+            "[yellow]    Links / redirects / cookies will use that origin. To silence this hint, set BASE_URL[/yellow]",
+        )
+        print(
+            f"[yellow]    explicitly: [bold]BASE_URL={csrf_derived}[/bold] (matches your existing CSRF_TRUSTED_ORIGINS).[/yellow]",
+        )
+        print()
+        return
 
-    return stopped
+    # BASE_URL was not set explicitly. The routes_util derivation gives one of
+    # three results, with very different risk profiles — show a tailored hint
+    # so new users coming from the 0.7.x single-domain world know whether the
+    # default is fine for them or needs attention.
+    if _bind_host_looks_like_ip(host):
+        # Real IP literal: subdomain routing can't work, URLs leak the IP.
+        # This is the most urgent case.
+        print(
+            f"[yellow][!] WARNING: BASE_URL is not set and BIND_ADDR resolves to an IP literal ({host}).[/yellow]",
+        )
+        print(
+            "[yellow]    Snapshot / admin / api URLs will be generated with the IP, and subdomain[/yellow]",
+        )
+        print(
+            "[yellow]    routing cannot work against an IP address. Set BASE_URL explicitly, e.g.[/yellow]",
+        )
+        print(
+            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
+        )
+        if config.USES_SUBDOMAIN_ROUTING:
+            print(
+                "[yellow]    Or switch SERVER_SECURITY_MODE to a one-domain mode if you can't run a hostname.[/yellow]",
+            )
+        print()
+    else:
+        # Loopback / wildcard bind. The routes_util default of
+        # http://archivebox.localhost:PORT works in a browser on the same
+        # machine, but anything else (reverse proxy, k8s ingress, LAN client)
+        # needs BASE_URL set. (Real hostnames can't reach this branch — the
+        # bind validator rejects them upfront.)
+        print(
+            "[yellow][!] BASE_URL is not set. Generated URLs will fall back to http://archivebox.localhost:<port>.[/yellow]",
+        )
+        print(
+            "[yellow]    That's fine for local browsing on this machine. Set BASE_URL when running behind[/yellow]",
+        )
+        print(
+            "[yellow]    a reverse proxy / ingress / public hostname, e.g.[/yellow]",
+        )
+        print(
+            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
+        )
+        print()
 
 
 @enforce_types
 def server(
     runserver_args: Iterable[str] | None = None,
     reload: bool = False,
-    init: bool = False,
     debug: bool = False,
     daemonize: bool = False,
     nothreading: bool = False,
+    createsuperuser: bool = False,
 ) -> None:
     """Run the ArchiveBox HTTP server"""
+    from archivebox.config.common import get_config
 
     config = get_config()
     runserver_args = list(runserver_args or (config.BIND_ADDR,))
 
-    if init:
-        from archivebox.cli.archivebox_init import init as archivebox_init
-
-        archivebox_init(quick=True)
-        print()
-
-    from archivebox.misc.checks import check_data_folder
-
-    check_data_folder()
-
     run_in_debug = config.DEBUG or debug or reload
     if debug or reload:
         os.environ["DEBUG"] = "True"
 
     from django.contrib.auth.models import User
 
+    if createsuperuser:
+        from archivebox.cli.archivebox_manage import manage
+
+        manage(args=["createsuperuser"])
+        print()
+
     if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
         print()
         print(
@@ -132,90 +259,149 @@ def server(
         print("      [green]archivebox manage createsuperuser[/green]")
         print()
 
-    host = "127.0.0.1"
-    port = "8000"
+    # First non-empty positional arg is the bind spec; otherwise inherit from
+    # config (which defaults to "127.0.0.1:8000"). _parse_and_validate_bind_spec
+    # hard-errors on hostnames so the rest of the server can assume a numeric
+    # bind host.
+    bind_spec = next((arg for arg in runserver_args if arg), "")
+    host, port = _parse_and_validate_bind_spec(bind_spec)
 
-    try:
-        host_and_port = [arg for arg in runserver_args if arg.replace(".", "").replace(":", "").isdigit()][0]
-        if ":" in host_and_port:
-            host, port = host_and_port.split(":")
+    if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1":
+        log_path = CONSTANTS.LOGS_DIR / "server.log"
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        daemon_env = os.environ.copy()
+        daemon_env["ARCHIVEBOX_SERVER_DAEMON_CHILD"] = "1"
+        daemon_cmd = [sys.executable, "-m", "archivebox", "server"]
+        if debug:
+            daemon_cmd.append("--debug")
+        if reload:
+            daemon_cmd.append("--reload")
+        if nothreading:
+            daemon_cmd.append("--nothreading")
+        daemon_cmd.extend(runserver_args)
+        with log_path.open("a", encoding="utf-8") as log_file:
+            proc = subprocess.Popen(
+                daemon_cmd,
+                cwd=os.getcwd(),
+                env=daemon_env,
+                stdin=subprocess.DEVNULL,
+                stdout=log_file,
+                stderr=log_file,
+                start_new_session=True,
+            )
+        deadline = time.monotonic() + 30
+        while time.monotonic() < deadline:
+            if proc.poll() is not None:
+                print(f"[red][X] ArchiveBox daemon server exited early with code {proc.returncode}. See {log_path}[/red]")
+                sys.exit(proc.returncode or 1)
+            try:
+                with socket.create_connection((host, int(port)), timeout=0.25):
+                    break
+            except OSError:
+                time.sleep(0.1)
         else:
-            if "." in host_and_port:
-                host = host_and_port
-            else:
-                port = host_and_port
-    except IndexError:
-        pass
+            print(f"[yellow][!] ArchiveBox daemon server pid={proc.pid} is still starting. See {log_path}[/yellow]")
+        return
+
+    os.environ["BIND_ADDR"] = f"{host}:{port}"
+    from archivebox.core.routes_util import get_base_url
+
+    base_url = get_base_url().rstrip("/")
+    admin_url = f"{base_url}/admin/"
 
     from archivebox.workers.supervisord_util import (
-        get_existing_supervisord_process,
-        get_worker,
-        stop_worker,
+        active_supervisord_runtime_components,
+        format_runtime_components,
         start_server_workers,
+        stop_existing_supervisord_process,
         is_port_in_use,
     )
-    from archivebox.machine.models import Machine, Process
-
-    machine = Machine.current()
-    supervisor = get_existing_supervisord_process()
-    stop_existing_background_runner(
-        machine=machine,
-        process_model=Process,
-        supervisor=supervisor,
-        stop_worker_fn=stop_worker,
+    from archivebox.machine.models import Process
+    from archivebox.core.takeover_util import (
+        command_owns_runtime_stack,
+        current_command,
+        foreground_runner_owner,
+        runtime_stack_owner,
+        standby_until_runtime_stack_needed,
     )
-    if supervisor:
-        stop_existing_server_workers(
-            supervisor=supervisor,
-            stop_worker_fn=stop_worker,
-            host=host,
-            port=port,
-        )
-
-    # Check if port is already in use
-    if is_port_in_use(host, int(port)):
-        print(f"[red][X] Error: Port {port} is already in use[/red]")
-        print(f"    Another process (possibly daphne or runserver) is already listening on {host}:{port}")
-        print("    Stop the conflicting process or choose a different port")
-        sys.exit(1)
-
-    supervisor = get_existing_supervisord_process()
-    if supervisor:
-        server_worker_name = "worker_runserver" if run_in_debug else "worker_daphne"
-        server_proc = get_worker(supervisor, server_worker_name)
-        server_state = server_proc.get("statename") if isinstance(server_proc, dict) else None
-        if server_state == "RUNNING":
-            runner_proc = get_worker(supervisor, "worker_runner")
-            runner_watch_proc = get_worker(supervisor, "worker_runner_watch")
-            runner_state = runner_proc.get("statename") if isinstance(runner_proc, dict) else None
-            runner_watch_state = runner_watch_proc.get("statename") if isinstance(runner_watch_proc, dict) else None
-            print("[red][X] Error: ArchiveBox server is already running[/red]")
-            print(
-                f"    [green]√[/green] Web server ({server_worker_name}) is RUNNING on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
-            )
-            if runner_state == "RUNNING":
-                print("    [green]√[/green] Background runner (worker_runner) is RUNNING")
-            if runner_watch_state == "RUNNING":
-                print("    [green]√[/green] Reload watcher (worker_runner_watch) is RUNNING")
-            print()
-            print("[yellow]To stop the existing server, run:[/yellow]")
-            print('    pkill -f "archivebox server"')
-            print("    pkill -f supervisord")
-            sys.exit(1)
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
 
     if run_in_debug:
         print("[green][+] Starting ArchiveBox webserver in DEBUG mode...[/green]")
     else:
         print("[green][+] Starting ArchiveBox webserver...[/green]")
     print(
-        f"    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
+        f"    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [dim]BIND_ADDR[/dim] [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
     )
     print(
-        f"    [green]>[/green] Log in to ArchiveBox Admin UI on [deep_sky_blue3][link=http://{host}:{port}/admin]http://{host}:{port}/admin[/link][/deep_sky_blue3]",
+        f"    [green]>[/green] Log in to ArchiveBox Admin UI on [dim]BASE_URL [/dim] [deep_sky_blue3][link={admin_url}]{admin_url}[/link][/deep_sky_blue3]",
     )
     print("    > Writing ArchiveBox error log to ./logs/errors.log")
     print()
-    start_server_workers(host=host, port=port, daemonize=daemonize, debug=run_in_debug, reload=reload, nothreading=nothreading)
+
+    # Reload config after we've set os.environ["BIND_ADDR"] above so the
+    # security-mode + base-url warnings see the effective values.
+    runtime_config = get_config()
+    _print_server_startup_warnings(runtime_config, host, port)
+    bind_url = f"http://{host}:{port}"
+    command = current_command(Process.TypeChoices.SERVER, data_dir=CONSTANTS.DATA_DIR, url=bind_url)
+
+    def still_owns_runtime_stack() -> bool:
+        from django.db import connections
+
+        try:
+            return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
+        finally:
+            connections.close_all()
+
+    shutdown_state = None
+    try:
+        with (
+            foreground_shutdown_signals() as shutdown_state,
+            foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"),
+        ):
+            while True:
+                standby_result = standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id) or foreground_runner_owner(
+                    data_dir=CONSTANTS.DATA_DIR,
+                    exclude_id=command.id,
+                )
+                takeover_components = active_supervisord_runtime_components(config=config)
+                if older_owner and takeover_components:
+                    print(
+                        "[yellow][*] Taking over "
+                        f"{format_runtime_components(takeover_components)} from older existing archivebox process (pid={older_owner.pid}).[/yellow]",
+                    )
+                stop_existing_supervisord_process()
+                if is_port_in_use(host, int(port)):
+                    print(f"[red][X] Error: Port {port} is already in use[/red]")
+                    print(f"    Another process outside this ArchiveBox runtime is listening on {host}:{port}")
+                    sys.exit(1)
+
+                result = start_server_workers(
+                    host=host,
+                    port=port,
+                    daemonize=False,
+                    debug=run_in_debug,
+                    reload=reload,
+                    nothreading=nothreading,
+                    keep_running=still_owns_runtime_stack,
+                    should_stop_supervisord=still_owns_runtime_stack,
+                    resumed_from_pid=standby_result.get("previous_owner_pid") if standby_result.get("resumed") else None,
+                )
+                if result == "interrupted":
+                    break
+                if not still_owns_runtime_stack():
+                    continue
+                if result == "exited":
+                    print("[yellow][*] Runtime stack exited while this parent is still leader; restarting...[/yellow]")
+                    continue
+                break
+    except KeyboardInterrupt:
+        pass
+    finally:
+        if not shutdown_state or not shutdown_state.signal_name:
+            command.mark_exited()
     print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")
 
 
@@ -224,8 +410,8 @@ def server(
 @click.option("--reload", is_flag=True, help="Enable auto-reloading when code or templates change")
 @click.option("--debug", is_flag=True, help="Enable DEBUG=True mode with more verbose errors")
 @click.option("--nothreading", is_flag=True, help="Force runserver to run in single-threaded mode")
-@click.option("--init", is_flag=True, help="Run a full archivebox init/upgrade before starting the server")
 @click.option("--daemonize", is_flag=True, help="Run the server in the background as a daemon")
+@click.option("--createsuperuser", is_flag=True, help="Run archivebox manage createsuperuser before starting the server")
 @docstring(server.__doc__)
 def main(**kwargs):
     server(**kwargs)

archivebox/cli/archivebox_shell.py

@@ -12,9 +12,10 @@ from archivebox.misc.util import docstring
 def shell(args: Iterable[str] = ()) -> None:
     """Enter an interactive ArchiveBox Django shell"""
 
-    from django.core.management import call_command
+    from django.core.management import call_command, get_commands
 
-    call_command("shell_plus", *args)
+    shell_command = "shell_plus" if "shell_plus" in get_commands() else "shell"
+    call_command(shell_command, *args)
 
 
 @click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))

archivebox/cli/archivebox_snapshot.py

@@ -35,10 +35,10 @@ from collections.abc import Iterable
 
 import rich_click as click
 from rich import print as rprint
-from django.db.models import Case, IntegerField, Q, Sum, When
-from django.db.models.functions import Coalesce
+from django.db.models import QuerySet
 
-from archivebox.cli.cli_utils import apply_filters
+SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp")
+SNAPSHOT_LIST_CHUNK_SIZE = 100
 
 
 # =============================================================================
@@ -67,7 +67,7 @@ def create_snapshots(
         TYPE_SNAPSHOT,
         TYPE_CRAWL,
     )
-    from archivebox.misc.util import validate_url_length
+    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot
     from archivebox.crawls.models import Crawl
@@ -112,7 +112,7 @@ def create_snapshots(
                 # Create snapshots for each URL in the crawl
                 for url in crawl.get_urls_list():
                     try:
-                        validate_url_length(url)
+                        validate_url(url)
                     except ValueError as err:
                         rprint(f"[red]Error creating snapshot: {err}[/red]", file=sys.stderr)
                         continue
@@ -135,7 +135,7 @@ def create_snapshots(
             elif record_type == TYPE_SNAPSHOT or record.get("url"):
                 # Input is a Snapshot or plain URL
                 if record.get("url"):
-                    validate_url_length(str(record["url"]))
+                    validate_url(str(record["url"]))
                 if tag and not record.get("tags"):
                     record["tags"] = tag
                 if status:
@@ -179,18 +179,63 @@ def create_snapshots(
 # =============================================================================
 
 
+def snapshot_filter_options(*, default_filter_type: str):
+    def decorate(func):
+        for decorator in reversed(
+            (
+                click.option("--status", "-s", help="Filter by status (queued, started, sealed)"),
+                click.option("--url__icontains", help="Filter by URL contains"),
+                click.option("--url__istartswith", help="Filter by URL starts with"),
+                click.option("--tag", "-t", help="Filter by tag name"),
+                click.option("--crawl-id", help="Filter by crawl ID"),
+                click.option("--limit", "-n", type=int, help="Limit number of results"),
+                click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at"),
+                click.option("--search", help="Search mode to use for positional query"),
+                click.option("--before", type=float, help="Only snapshots bookmarked before timestamp"),
+                click.option("--after", type=float, help="Only snapshots bookmarked after timestamp"),
+                click.option(
+                    "--filter-type",
+                    "-f",
+                    type=click.Choice(SNAPSHOT_FILTER_TYPE_CHOICES),
+                    default=default_filter_type,
+                    help="Type of pattern matching to use for positional filters",
+                ),
+                click.argument("filter_patterns", nargs=-1),
+            ),
+        ):
+            func = decorator(func)
+        return func
+
+    return decorate
+
+
+def snapshot_output_options(func):
+    for decorator in reversed(
+        (
+            click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title"),
+            click.option("--json", "as_json", is_flag=True, help="Print output as a JSON array"),
+            click.option("--html", "as_html", is_flag=True, help="Print output as HTML"),
+            click.option("--with-headers", is_flag=True, help="Include column headers in structured output"),
+        ),
+    ):
+        func = decorator(func)
+    return func
+
+
+def build_snapshot_queryset(
+    **kwargs,
+) -> QuerySet:
+    from archivebox.core.models import Snapshot
+
+    return Snapshot.objects.order_by("-created_at").search(**kwargs)
+
+
 def list_snapshots(
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
     csv: str | None = None,
+    as_json: bool = False,
+    as_html: bool = False,
     with_headers: bool = False,
-    search: str | None = None,
-    query: str | None = None,
+    **kwargs,
 ) -> int:
     """
     List Snapshots as JSONL with optional filters.
@@ -199,142 +244,71 @@ def list_snapshots(
         0: Success (even if no results)
     """
     from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import Snapshot
-    from archivebox.search import (
-        get_default_search_mode,
-        get_search_mode,
-        prioritize_metadata_matches,
-        query_search_index,
-    )
 
-    if with_headers and not csv:
-        rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr)
+    output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
+    if output_formats > 1:
+        rprint("[red]Choose only one output format: --csv, --json, or --html[/red]", file=sys.stderr)
+        return 2
+    if with_headers and not output_formats:
+        rprint("[red]--with-headers requires --csv, --json, or --html[/red]", file=sys.stderr)
         return 2
 
-    is_tty = sys.stdout.isatty() and not csv
+    is_tty = sys.stdout.isatty() and not output_formats
 
-    queryset = Snapshot.objects.order_by("-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "url__icontains": url__icontains,
-        "url__istartswith": url__istartswith,
-        "crawl_id": crawl_id,
-    }
-    queryset = apply_filters(queryset, filter_kwargs)
-
-    # Tag filter requires special handling (M2M)
-    if tag:
-        queryset = queryset.filter(tags__name__iexact=tag)
-
-    query = (query or "").strip()
-    if query:
-        metadata_qs = queryset.filter(
-            Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query),
-        )
-        requested_search_mode = (search or "").strip().lower()
-        if requested_search_mode == "content":
-            requested_search_mode = "contents"
-        search_mode = get_default_search_mode() if not requested_search_mode else get_search_mode(requested_search_mode)
-
-        if search_mode == "meta":
-            queryset = metadata_qs
-        else:
-            try:
-                deep_qsearch = None
-                if search_mode == "deep":
-                    qsearch = query_search_index(query, search_mode="contents")
-                    deep_qsearch = query_search_index(query, search_mode="deep")
-                else:
-                    qsearch = query_search_index(query, search_mode=search_mode)
-                queryset = prioritize_metadata_matches(
-                    queryset,
-                    metadata_qs,
-                    qsearch,
-                    deep_queryset=deep_qsearch,
-                    ordering=("-created_at",) if not sort else None,
-                )
-            except Exception as err:
-                rprint(
-                    f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
-                    file=sys.stderr,
-                )
-                queryset = metadata_qs
-
-    if sort:
-        queryset = queryset.order_by(sort)
-
-    if not is_tty:
-        if limit:
-            limited_ids = list(queryset.values_list("id", flat=True)[:limit])
-            preserved_order = Case(
-                *(When(id=snapshot_id, then=position) for position, snapshot_id in enumerate(limited_ids)),
-                output_field=IntegerField(),
-            )
-            queryset = Snapshot.objects.filter(id__in=limited_ids).order_by(preserved_order)
-        queryset = queryset.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)).prefetch_related("tags")
-    elif limit:
-        queryset = queryset[:limit]
+    try:
+        queryset = build_snapshot_queryset(**kwargs)
+    except ValueError as err:
+        rprint(f"[red]{err}[/red]", file=sys.stderr)
+        return 2
 
     count = 0
+    if as_json:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_json(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    if as_html:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_html(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
     if csv:
         cols = [col.strip() for col in csv.split(",") if col.strip()]
         if not cols:
             rprint("[red]No CSV columns provided[/red]", file=sys.stderr)
             return 2
-        rows: list[str] = []
         if with_headers:
-            rows.append(",".join(cols))
-        simple_cols = {
-            "id",
-            "crawl_id",
-            "url",
-            "title",
-            "timestamp",
-            "depth",
-            "status",
-            "fs_version",
-            "bookmarked_at",
-            "created_at",
-            "modified_at",
-            "retry_at",
-            "downloaded_at",
-        }
-        from archivebox.misc.util import to_json
-
-        for snapshot in queryset.iterator(chunk_size=500):
-            if set(cols).issubset(simple_cols):
-                rows.append(
-                    ",".join(
-                        to_json(
-                            value.isoformat() if hasattr((value := getattr(snapshot, col, "")), "isoformat") else value,
-                            indent=None,
-                        )
-                        for col in cols
-                    ),
-                )
-            else:
-                rows.append(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write(",".join(cols))
+            sys.stdout.write("\n")
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+            sys.stdout.write(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write("\n")
             count += 1
-        output = "\n".join(rows)
-        if output:
-            sys.stdout.write(output)
-            if not output.endswith("\n"):
-                sys.stdout.write("\n")
         rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
         return 0
 
-    for snapshot in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(snapshot.status, "dim")
-            rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
-        else:
+    if not is_tty:
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
             write_record(snapshot.to_json())
+            count += 1
+        rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    for snapshot in queryset.iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+        status_color = {
+            "queued": "yellow",
+            "started": "blue",
+            "sealed": "green",
+        }.get(snapshot.status, "dim")
+        rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
         count += 1
 
     rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
@@ -381,22 +355,34 @@ def update_snapshots(
         try:
             snapshot = Snapshot.objects.get(id=snapshot_id)
 
-            # Apply updates from CLI flags (override stdin values)
             if status:
-                snapshot.status = status
-                snapshot.retry_at = timezone.now()
+                if status not in Snapshot.StatusChoices.values:
+                    rprint(f"[red]Invalid snapshot status: {status}[/red]", file=sys.stderr)
+                    continue
+                if status == Snapshot.StatusChoices.SEALED:
+                    snapshot.cancel()
+                elif status == Snapshot.StatusChoices.PAUSED:
+                    snapshot.pause()
+                elif status == Snapshot.StatusChoices.QUEUED:
+                    if snapshot.status == Snapshot.StatusChoices.PAUSED:
+                        snapshot.resume()
+                    else:
+                        snapshot.update_and_requeue(status=Snapshot.StatusChoices.QUEUED, retry_at=timezone.now())
+                elif status == Snapshot.StatusChoices.STARTED:
+                    snapshot.update_and_requeue(status=Snapshot.StatusChoices.STARTED, retry_at=timezone.now())
             if tag:
-                # Add tag to existing tags
-                snapshot.save()  # Ensure saved before M2M
                 from archivebox.core.models import Tag
 
                 tag_obj, _ = Tag.objects.get_or_create(name=tag)
                 snapshot.tags.add(tag_obj)
+                snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
 
-            snapshot.save()
+            if not status and not tag:
+                snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
             updated_count += 1
 
             if not is_tty:
+                snapshot.refresh_from_db()
                 write_record(snapshot.to_json())
 
         except Snapshot.DoesNotExist:
@@ -481,46 +467,11 @@ def create_cmd(urls: tuple, tag: str, status: str, depth: int):
 
 
 @main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def list_cmd(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-):
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def list_cmd(**kwargs):
     """List Snapshots as JSONL."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))
 
 
 @main.command("update")

archivebox/cli/archivebox_snapshot_compat.py

@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox snapshot"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_snapshot import create_snapshots
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--depth", "-d", type=int, default=0, help="Crawl depth (default: 0)")
-@click.argument("urls", nargs=-1)
-def main(tag: str, status: str, depth: int, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox snapshot URL...` entrypoint."""
-    sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth))
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_status.py

@@ -8,7 +8,7 @@ import rich_click as click
 from rich import print
 
 from archivebox.misc.util import enforce_types, docstring
-from archivebox.config import DATA_DIR, CONSTANTS
+from archivebox.config import CONSTANTS
 from archivebox.config.common import get_config
 from archivebox.misc.system import get_dir_size
 from archivebox.misc.logging_util import printable_filesize
@@ -18,7 +18,7 @@ MAX_STATUS_FS_DIR_SCAN = 5000
 
 
 @enforce_types
-def status(out_dir: Path = DATA_DIR) -> None:
+def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
     """Print out some info and statistics about the archive collection"""
 
     from django.contrib.auth import get_user_model
@@ -38,7 +38,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
 
     snapshots_qs = Snapshot.objects.all()
     num_sql_links = snapshots_qs.count()
-    archive_dir = config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
     legacy_snapshot_dirs = []
     if archive_dir.exists():
         legacy_snapshot_dirs = [
@@ -48,7 +48,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
     print(f"    > JSON Link Details: {len(legacy_snapshot_dirs)} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)")
     print()
     print("[green]\\[*] Scanning archive data directories...[/green]")
-    users_dir = config.USERS_DIR
+    users_dir = CONSTANTS.USERS_DIR
     scan_roots = [root for root in (archive_dir, users_dir) if root.exists()]
     scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(archive_dir)
     print(f"[yellow]   {scan_roots_display}[/yellow]")
@@ -61,7 +61,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
             num_dirs += root_dirs
             num_files += root_files
     else:
-        num_bytes = ArchiveResult.objects.aggregate(total=Coalesce(Sum("output_size"), 0))["total"] or 0
+        num_bytes = snapshots_qs.aggregate(total=Coalesce(Sum("output_size"), 0))["total"] or 0
         num_dirs = 0
         num_files = ArchiveResult.objects.exclude(output_files__in=["", "{}"]).count()
     size = printable_filesize(num_bytes)
@@ -105,7 +105,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
 
     if num_indexed:
         print("    [violet]Hint:[/violet] You can list snapshots by status like so:")
-        print("        [green]archivebox list --status=<status>  (e.g. archived, queued, etc.)[/green]")
+        print("        [green]archivebox list --status=<status>  (e.g. sealed, queued, etc.)[/green]")
 
     if orphaned_dirs:
         print("    [violet]Hint:[/violet] To automatically import orphaned data directories into the main index, run:")
@@ -130,7 +130,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
         print("        [green]archivebox manage createsuperuser[/green]")
 
     print()
-    recent_snapshots = snapshots_qs.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)).order_by(
+    recent_snapshots = snapshots_qs.order_by(
         "-downloaded_at",
         "-modified_at",
     )[:10]
@@ -141,7 +141,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
             (
                 "[grey53] "
                 f"   > {str(snapshot.downloaded_at)[:16]} "
-                f"[{snapshot.num_outputs} {('X', '√')[snapshot.status == Snapshot.StatusChoices.SEALED]} {printable_filesize(snapshot.output_size_sum or 0)}] "
+                f"[{snapshot.num_outputs} {('X', '√')[snapshot.status == Snapshot.StatusChoices.SEALED]} {printable_filesize(snapshot.output_size or 0)}] "
                 f'"{snapshot.title}": {snapshot.url}'
                 "[/grey53]"
             )[: config.TERM_WIDTH],

archivebox/cli/archivebox_tag.py

@@ -36,7 +36,7 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 
 
 # =============================================================================

archivebox/cli/archivebox_version.py

@@ -29,10 +29,8 @@ def _format_binary_abspath(
 
     candidate_bases: tuple[tuple[Path, str], ...] = (
         (pwd, "./"),
-        (lib_dir, "LIB_DIR/"),
-        (Path(os.environ.get("LIB_DIR", "")), "LIB_DIR/") if os.environ.get("LIB_DIR") else (Path(), ""),
+        (lib_dir, "ABXPKG_LIB_DIR/"),
         (personas_dir, "PERSONAS_DIR/"),
-        (Path(os.environ.get("PERSONAS_DIR", "")), "PERSONAS_DIR/") if os.environ.get("PERSONAS_DIR") else (Path(), ""),
         (home, "~/"),
     )
 
@@ -48,8 +46,8 @@ def _format_binary_abspath(
             relative_str = relative.as_posix()
             if prefix == "./":
                 return "." if not relative_str else f"./{relative_str}"
-            if prefix == "LIB_DIR/":
-                return "LIB_DIR" if not relative_str else f"LIB_DIR/{relative_str}"
+            if prefix == "ABXPKG_LIB_DIR/":
+                return "ABXPKG_LIB_DIR" if not relative_str else f"ABXPKG_LIB_DIR/{relative_str}"
             if prefix == "PERSONAS_DIR/":
                 return "PERSONAS_DIR" if not relative_str else f"PERSONAS_DIR/{relative_str}"
             return "~" if not relative_str else f"~/{relative_str}"
@@ -60,10 +58,10 @@ def _format_binary_abspath(
 def _render_binary_abspath(abspath: str):
     from rich.text import Text
 
-    if abspath.startswith("LIB_DIR/"):
-        return Text.assemble(("LIB_DIR", "bright_blue"), (abspath.removeprefix("LIB_DIR"), "green"))
-    if abspath == "LIB_DIR":
-        return Text("LIB_DIR", style="bright_blue")
+    if abspath.startswith("ABXPKG_LIB_DIR/"):
+        return Text.assemble(("ABXPKG_LIB_DIR", "bright_blue"), (abspath.removeprefix("ABXPKG_LIB_DIR"), "green"))
+    if abspath == "ABXPKG_LIB_DIR":
+        return Text("ABXPKG_LIB_DIR", style="bright_blue")
     if abspath.startswith("PERSONAS_DIR/"):
         return Text.assemble(("PERSONAS_DIR", "medium_purple"), (abspath.removeprefix("PERSONAS_DIR"), "green"))
     if abspath == "PERSONAS_DIR":
@@ -79,6 +77,79 @@ def _render_binary_abspath(abspath: str):
     return Text(abspath, style="green")
 
 
+def _build_binary_table(rows: list[dict[str, object]]):
+    from rich import box
+    from rich.table import Table
+
+    table = Table(title="Binary Dependencies", box=box.SIMPLE_HEAVY, expand=True)
+    table.add_column("Binary", no_wrap=True, max_width=28)
+    table.add_column("Plugin", no_wrap=True, max_width=24)
+    table.add_column("State", no_wrap=True, width=8)
+    table.add_column("Status", justify="center", no_wrap=True, width=6)
+    table.add_column("Version", no_wrap=True, width=16)
+    table.add_column("Provider", no_wrap=True, width=8)
+    table.add_column("Path", overflow="fold", ratio=1)
+    for row in rows:
+        table.add_row(
+            str(row["binary"]),
+            str(row["plugin"]),
+            str(row["state"]),
+            str(row["status"]),
+            str(row["version"]),
+            str(row["provider"]),
+            row["path"],
+            style=str(row.get("style") or ""),
+        )
+    return table
+
+
+def _print_binary_row(prnt, row: dict[str, object]) -> None:
+    prnt(
+        "",
+        str(row["binary"]).ljust(28),
+        row["status"],
+        str(row["plugin"]).ljust(24),
+        str(row["state"]).ljust(8),
+        str(row["version"]).ljust(16),
+        str(row["provider"]).ljust(8),
+        row["path"],
+        overflow="ignore",
+        crop=False,
+        style=str(row.get("style") or ""),
+    )
+
+
+def _binary_record_matches_runtime(installed, lib_dir: Path) -> bool:
+    if not installed or not installed.is_valid or not installed.version:
+        return False
+    try:
+        abspath = Path(installed.abspath).expanduser().resolve(strict=False)
+        if not abspath.exists():
+            return False
+        if installed.binprovider not in {"", "env", "apt", "brew"}:
+            abspath.relative_to(lib_dir)
+    except (OSError, ValueError):
+        return False
+    return True
+
+
+def _binary_row_dedupe_key(
+    *,
+    display_name: str,
+    valid: bool,
+    version: str,
+    provider: str,
+    abspath: str,
+) -> tuple[str, str, str, str]:
+    if not valid:
+        return (display_name, "", "", "")
+    try:
+        resolved_abspath = Path(abspath).expanduser().resolve(strict=False).as_posix()
+    except Exception:
+        resolved_abspath = abspath
+    return (display_name, provider, version, resolved_abspath)
+
+
 @enforce_types
 def version(
     quiet: bool = False,
@@ -95,13 +166,14 @@ def version(
 
     from rich.panel import Panel
     from rich.console import Console
+    from rich.live import Live
 
     from archivebox.config import CONSTANTS
     from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
     from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID, IN_DOCKER
     from archivebox.config.paths import get_data_locations, get_code_locations
     from archivebox.misc.logging_util import printable_folder_status
-    from archivebox.config.common import get_config
+    from archivebox.config.common import get_config, normalize_runtime_config
 
     console = Console()
     prnt = console.print
@@ -134,7 +206,7 @@ def version(
     try:
         DATA_DIR_STAT = CONSTANTS.DATA_DIR.stat()
         prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
             f"FS_UID={DATA_DIR_STAT.st_uid}:{DATA_DIR_STAT.st_gid}",
             f"FS_PERMS={config.OUTPUT_PERMISSIONS}",
             f"FS_ATOMIC={config.ENFORCE_ATOMIC_WRITES}",
@@ -142,7 +214,7 @@ def version(
         )
     except Exception:
         prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
         )
 
     prnt(
@@ -155,7 +227,13 @@ def version(
     )
     prnt()
 
-    if not (os.access(config.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
+    in_data_dir = os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)
+    if isinstance(binaries, str):
+        requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
+    else:
+        requested_names = {name for name in (binaries or ()) if name}
+
+    if not in_data_dir:
         PANEL_TEXT = "\n".join(
             (
                 "",
@@ -175,93 +253,219 @@ def version(
                 subtitle="Full version info is only available when inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
             ),
         )
+    prnt()
+
+    if not in_data_dir and not requested_names:
+        prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
+        prnt("", "[grey53]Dependency checks require an initialized collection DATA_DIR.[/grey53]")
+        prnt()
+        prnt("[deep_sky_blue3][i] Code locations:[/deep_sky_blue3]")
+        try:
+            for name, path in get_code_locations().items():
+                if isinstance(name, str) and isinstance(path, dict):
+                    prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
+        except Exception as e:
+            prnt(f"  [red]Error getting code locations: {e}[/red]")
+        prnt()
+        prnt("[red][i] Data locations:[/red] (not in a data directory)")
         prnt()
         return []
 
     prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
     failures = []
+    seen_failures: set[str] = set()
+    seen_rows: set[tuple[str, str, str, str]] = set()
 
-    # Setup Django before importing models
+    from archivebox.plugins.discovery import get_enabled_plugins
+    from abx_dl.config import get_required_binary_requests
+    from abx_dl.dependencies import load_binary
+    from abx_dl.models import discover_plugins, filter_plugins
+
+    plugins = discover_plugins(runtime="archivebox")
+    enabled_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
+    enabled_plugin_names = set(enabled_plugins)
+    runtime_config = normalize_runtime_config(config.for_crawl(), json_safe=False)
+    derived_config: dict[str, object] = {}
+    db_binaries = {}
+    db_available = False
+    if in_data_dir:
+        try:
+            from archivebox.config.django import setup_django
+
+            setup_django()
+
+            from archivebox.machine.models import Machine, Binary
+
+            machine = Machine.current()
+            derived_config = normalize_runtime_config(machine.config, json_safe=False)
+            for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
+                if _binary_record_matches_runtime(binary, config.ABXPKG_LIB_DIR):
+                    db_binaries.setdefault(binary.name, binary)
+            db_available = True
+
+        except Exception as e:
+            prnt()
+            prnt("", f"[yellow]Warning: Could not query collection binary records, falling back to abxpkg state: {e}[/yellow]")
+
+    rows: list[dict[str, object]] = []
+    any_rows = False
+    any_available = False
+    compact_paths = console.is_terminal
+    live_enabled = console.is_terminal
+    live_cm = Live(_build_binary_table(rows), console=console, refresh_per_second=8) if live_enabled else None
+    if not live_enabled:
+        prnt("", "Binary".ljust(28), "Status", "Plugin".ljust(24), "State".ljust(8), "Version".ljust(16), "Provider".ljust(8), "Path")
+
+    def emit_row(row: dict[str, object]) -> None:
+        rows.append(row)
+        if live_cm is not None:
+            live_cm.update(_build_binary_table(rows), refresh=True)
+        else:
+            _print_binary_row(prnt, row)
+
+    if live_cm is not None:
+        live_cm.start()
     try:
-        from archivebox.config.django import setup_django
-
-        setup_django()
-
-        from archivebox.machine.models import Machine, Binary
-
-        machine = Machine.current()
-
-        if isinstance(binaries, str):
-            requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
-        else:
-            requested_names = {name for name in (binaries or ()) if name}
-
-        db_binaries: dict[str, Binary] = {}
-        for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
-            db_binaries.setdefault(binary.name, binary)
-
-        all_binary_names = sorted(requested_names or set(db_binaries.keys()))
-
-        if not all_binary_names:
-            prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
-        else:
-            any_available = False
-            compact_paths = console.is_terminal
-            for name in all_binary_names:
-                if requested_names and name not in requested_names:
+        for plugin_name, plugin in plugins.items():
+            plugin_enabled = plugin_name in enabled_plugin_names
+            logical_records = get_required_binary_requests(
+                plugin,
+                plugin.config.required_binaries,
+                overrides=runtime_config,
+                derived_overrides=derived_config,
+                run_output_dir=CONSTANTS.DATA_DIR,
+            )
+            actual_records = get_required_binary_requests(
+                plugin,
+                plugin.config.required_binaries,
+                overrides=runtime_config,
+                derived_overrides=derived_config,
+                run_output_dir=CONSTANTS.DATA_DIR,
+                logical_names=False,
+            )
+            for logical_record, actual_record in zip(logical_records, actual_records, strict=False):
+                logical_name = str(logical_record["name"])
+                actual_name = str(actual_record["name"])
+                display_name = Path(actual_name).expanduser().name if ("/" in actual_name or actual_name.startswith("~")) else logical_name
+                if (
+                    requested_names
+                    and logical_name not in requested_names
+                    and actual_name not in requested_names
+                    and display_name not in requested_names
+                ):
                     continue
 
-                installed = db_binaries.get(name)
-                if installed and installed.is_valid:
-                    display_name = Path(name).expanduser().name if ("/" in name or name.startswith("~")) else name
+                any_rows = True
+                installed = db_binaries.get(logical_name) if db_available else None
+                if _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
+                    abspath = installed.abspath
+                    version_str = (installed.version or "unknown")[:15]
+                    provider = (installed.binprovider or "env")[:8]
+                    valid = True
+                else:
+                    loaded = load_binary(actual_record)
+                    abspath = str(loaded.loaded_abspath or "")
+                    version_str = str(loaded.loaded_version or "unknown")[:15]
+                    provider = (loaded.loaded_binprovider.name if loaded.loaded_binprovider else "env")[:8]
+                    valid = loaded.is_valid
+
+                if valid:
                     display_path = (
                         _format_binary_abspath(
-                            installed.abspath,
+                            abspath,
                             pwd=Path.cwd(),
-                            lib_dir=config.LIB_DIR,
-                            personas_dir=config.PERSONAS_DIR,
+                            lib_dir=config.ABXPKG_LIB_DIR,
+                            personas_dir=CONSTANTS.PERSONAS_DIR,
                             home=Path.home(),
                         )
                         if compact_paths
-                        else installed.abspath
+                        else abspath
                     )
                     rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
-                    version_str = (installed.version or "unknown")[:15]
-                    provider = (installed.binprovider or "env")[:8]
-                    prnt(
-                        "",
-                        "[green]√[/green]",
-                        "",
-                        display_name.ljust(18),
-                        version_str.ljust(16),
-                        provider.ljust(8),
-                        rendered_path,
-                        overflow="ignore",
-                        crop=False,
-                    )
+                    status = "[green]√[/green]" if plugin_enabled else "[grey53]-[/grey53]"
                     any_available = True
-                    continue
+                else:
+                    rendered_path = "[grey53]not installed[/grey53]"
+                    status = "[red]X[/red]" if plugin_enabled else "[grey53]-[/grey53]"
+                    if plugin_enabled and display_name not in seen_failures:
+                        failures.append(display_name)
+                        seen_failures.add(display_name)
 
-                status = (
-                    "[grey53]not recorded[/grey53]" if name in requested_names and installed is None else "[grey53]not installed[/grey53]"
+                row_key = _binary_row_dedupe_key(
+                    display_name=display_name,
+                    valid=valid,
+                    version=version_str if valid else "-",
+                    provider=provider if valid else "-",
+                    abspath=abspath,
                 )
-                prnt("", "[red]X[/red]", "", name.ljust(18), status, overflow="ignore", crop=False)
-                failures.append(name)
+                if row_key in seen_rows:
+                    continue
+                seen_rows.add(row_key)
 
-            if not any_available:
-                prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
+                emit_row(
+                    {
+                        "plugin": plugin_name,
+                        "state": "enabled" if plugin_enabled else "disabled",
+                        "status": status,
+                        "binary": display_name,
+                        "version": version_str if valid else "-",
+                        "provider": provider if valid else "-",
+                        "path": rendered_path,
+                        "style": "" if plugin_enabled else "dim",
+                    },
+                )
 
-        # Show hint if no binaries are installed yet
-        has_any_installed = Binary.objects.filter(machine=machine).exclude(abspath="").exists()
-        if not has_any_installed:
-            prnt()
-            prnt("", "[grey53]Run [green]archivebox install[/green] to detect and install dependencies.[/grey53]")
+        if db_available:
+            for binary_name, installed in db_binaries.items():
+                if requested_names and binary_name not in requested_names:
+                    continue
+                abspath = installed.abspath
+                version_str = (installed.version or "unknown")[:15]
+                provider = (installed.binprovider or "env")[:8]
+                row_key = _binary_row_dedupe_key(
+                    display_name=binary_name,
+                    valid=True,
+                    version=version_str,
+                    provider=provider,
+                    abspath=abspath,
+                )
+                if row_key in seen_rows:
+                    continue
+                seen_rows.add(row_key)
 
-    except Exception as e:
-        # Handle database errors gracefully (locked, missing, etc.)
-        prnt()
-        prnt("", f"[yellow]Warning: Could not query binaries from database: {e}[/yellow]")
-        prnt("", "[grey53]Run [green]archivebox init[/green] and [green]archivebox install[/green] to set up dependencies.[/grey53]")
+                display_path = (
+                    _format_binary_abspath(
+                        abspath,
+                        pwd=Path.cwd(),
+                        lib_dir=config.ABXPKG_LIB_DIR,
+                        personas_dir=CONSTANTS.PERSONAS_DIR,
+                        home=Path.home(),
+                    )
+                    if compact_paths
+                    else abspath
+                )
+                emit_row(
+                    {
+                        "plugin": "(database)",
+                        "state": "detected",
+                        "status": "[green]√[/green]",
+                        "binary": binary_name,
+                        "version": version_str,
+                        "provider": provider,
+                        "path": _render_binary_abspath(display_path) if compact_paths else display_path,
+                        "style": "dim",
+                    },
+                )
+                any_rows = True
+                any_available = True
+    finally:
+        if live_cm is not None:
+            live_cm.stop()
+
+    if not any_rows:
+        prnt("", "[grey53]No required binaries declared for discovered plugins.[/grey53]")
+    elif not any_available:
+        prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
 
     if not binaries:
         # Show code and data locations
@@ -275,7 +479,7 @@ def version(
             prnt(f"  [red]Error getting code locations: {e}[/red]")
 
         prnt()
-        if os.access(config.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
+        if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
             prnt("[bright_yellow][i] Data locations:[/bright_yellow]")
             try:
                 for name, path in get_data_locations().items():
@@ -320,9 +524,7 @@ def version(
 )
 @docstring(version.__doc__)
 def main(**kwargs):
-    failures = version(**kwargs)
-    if failures:
-        raise SystemExit(1)
+    version(**kwargs)
 
 
 if __name__ == "__main__":

archivebox/config/__init__.py

@@ -3,9 +3,21 @@
 __package__ = "archivebox.config"
 __order__ = 200
 
-from .paths import (
-    PACKAGE_DIR,
-    DATA_DIR,
-)
-from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR  # noqa
-from .version import VERSION  # noqa
+
+def __getattr__(name: str):
+    if name in ("CONSTANTS", "CONSTANTS_CONFIG"):
+        from .constants import CONSTANTS, CONSTANTS_CONFIG
+
+        return {"CONSTANTS": CONSTANTS, "CONSTANTS_CONFIG": CONSTANTS_CONFIG}[name]
+    if name in ("PACKAGE_DIR", "DATA_DIR"):
+        from .paths import PACKAGE_DIR, DATA_DIR
+
+        return {"PACKAGE_DIR": PACKAGE_DIR, "DATA_DIR": DATA_DIR}[name]
+    if name == "VERSION":
+        from .version import VERSION
+
+        return VERSION
+    raise AttributeError(name)
+
+
+__all__ = ("CONSTANTS", "CONSTANTS_CONFIG", "PACKAGE_DIR", "DATA_DIR", "VERSION")

archivebox/config/collection.py

@@ -1,37 +1,297 @@
 __package__ = "archivebox.config"
 
+import io
+import json
 import os
-
-from benedict import benedict
+from collections.abc import Mapping
+from typing import Any
 
 from archivebox.config.constants import CONSTANTS
 from archivebox.config.configset import CaseConfigParser
+from archivebox.misc.logging import AttrDict
 
 
-def write_config_file(config: dict[str, str]) -> benedict:
-    """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
+CONFIG_FILE_HEADER = (
+    "# This is the config file for your ArchiveBox collection.\n"
+    "#\n"
+    "# You can add options here manually in INI format, or automatically by running:\n"
+    "#    archivebox config --set KEY=VALUE\n"
+    "#\n"
+    "# This file is kept in sync 1:1 with Machine.config in the index DB —\n"
+    "# editing either side propagates to the other. ``archivebox init`` reads\n"
+    "# this file on startup; the admin Machine.config editor writes both.\n"
+    "#\n"
+    "# Full reference:\n"
+    "#    https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration\n"
+    "\n"
+)
 
+
+# Recursion guard for the bidirectional file<->DB mirror. Bidirectional sync
+# means a write on one side always triggers an update on the other; without
+# this flag, ``Machine.save -> mirror to file -> write_config_file -> mirror
+# back to Machine -> Machine.save -> ...`` would loop forever. Module-level
+# scalar is fine here — daphne handles each request on its own thread and the
+# admin/CLI write paths are inherently serialized through the DB.
+_MIRROR_IN_PROGRESS: bool = False
+# One-time-per-process startup sync, gated so subsequent ``Machine.current()``
+# calls collapse to a single boolean check.
+_INITIAL_SYNC_DONE: bool = False
+
+
+def _coerce_to_str_dict(config: Any) -> dict[str, str]:
+    """Project an arbitrary config payload to flat ``{UPPER_KEY: str}`` form.
+
+    INI files only round-trip strings, so we normalize Machine.config values
+    to strings on the way out and accept the same shape on the way back.
+    Pydantic re-coerces types at read time inside ``get_config``.
+
+    Composite values (``dict`` / ``list`` / ``tuple``) are JSON-encoded so
+    they round-trip back through pydantic-settings — ``str(some_dict)``
+    would produce Python's repr (``{'k': 'v'}`` with single quotes), and
+    pydantic-settings refuses to parse that as a ``dict`` field, causing
+    e.g. ``ABX_INSTALL_CACHE`` to crash with ``ValidationError: Input
+    should be a valid dictionary``.
+    """
+    if not config:
+        return {}
+    if not isinstance(config, Mapping):
+        return {}
+    flat: dict[str, str] = {}
+    for key, value in config.items():
+        upper_key = str(key).upper()
+        if value is None:
+            flat[upper_key] = ""
+        elif isinstance(value, (dict, list, tuple)):
+            flat[upper_key] = json.dumps(value, default=str)
+        else:
+            flat[upper_key] = str(value)
+    return flat
+
+
+def _load_file_config_dict() -> tuple[dict[str, str], float | None]:
+    """Return ``(flat_dict, mtime)`` for ``ArchiveBox.conf`` (``({}, None)`` if missing)."""
+    config_path = CONSTANTS.CONFIG_FILE
+    try:
+        mtime = config_path.stat().st_mtime
+    except FileNotFoundError:
+        return {}, None
+    parser = CaseConfigParser()
+    parser.read(config_path)
+    flat = {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+    return flat, mtime
+
+
+def _resolve_section_for_key(key: str, config_sections, plugin_configs) -> str:
+    for section in config_sections.values():
+        if key in type(section).model_fields:
+            return section.toml_section_header
+    for schema in plugin_configs.values():
+        if "properties" in schema and key in schema["properties"]:
+            return "PLUGINS"
+    # Unknown / user-defined keys land in SERVER_CONFIG so we never lose them
+    # (the previous code raised here, which was fine for the CLI-only path but
+    # would break the mirror as soon as anyone added a plugin-tunable that
+    # this process hasn't loaded a schema for).
+    return "SERVER_CONFIG"
+
+
+def _render_config_file_content(config: dict[str, str]) -> str:
+    """Render a flat config dict to INI text, grouped by inferred section."""
     from archivebox.config.common import get_all_configs
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
+
+    config_sections = get_all_configs()
+    plugin_configs = discover_plugin_configs()
+
+    parser = CaseConfigParser()
+    for key, val in sorted(config.items()):
+        section = _resolve_section_for_key(key, config_sections, plugin_configs)
+        if section not in parser:
+            parser[section] = {}
+        parser[section][key] = "" if val is None else str(val)
+
+    buf = io.StringIO()
+    buf.write(CONFIG_FILE_HEADER)
+    parser.write(buf)
+    return buf.getvalue()
+
+
+def _write_file_if_changed(content: str) -> bool:
+    """Atomic-write ``ArchiveBox.conf`` only when contents actually differ.
+
+    Skipping unchanged writes is the difference between every Machine.save in
+    a hot autodetection loop costing one disk write vs. zero.
+    """
     from archivebox.misc.system import atomic_write
 
-    CONFIG_HEADER = """# This is the config file for your ArchiveBox collection.
-    #
-    # You can add options here manually in INI format, or automatically by running:
-    #    archivebox config --set KEY=VALUE
-    #
-    # If you modify this file manually, make sure to update your archive after by running:
-    #    archivebox init
-    #
-    # A list of all possible config with documentation and examples can be found here:
-    #    https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration
+    config_path = CONSTANTS.CONFIG_FILE
+    try:
+        existing = config_path.read_text(encoding="utf-8") if config_path.exists() else None
+    except OSError:
+        existing = None
+    if existing == content:
+        return False
+    atomic_write(config_path, content)
+    return True
 
+
+def mirror_machine_config_to_file(config: Any) -> None:
+    """Rewrite ``ArchiveBox.conf`` so it mirrors ``Machine.config`` exactly.
+
+    Called from ``Machine.save`` after the row is committed. Recursion-guarded
+    so the matching write_config_file -> Machine.save bounce doesn't loop.
     """
+    global _MIRROR_IN_PROGRESS
+    if _MIRROR_IN_PROGRESS:
+        return
+    _MIRROR_IN_PROGRESS = True
+    try:
+        flat = _coerce_to_str_dict(config)
+        _write_file_if_changed(_render_config_file_content(flat))
+    finally:
+        _MIRROR_IN_PROGRESS = False
+
+
+def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
+    """Inverse of ``_coerce_to_str_dict``: decode complex INI values to native.
+
+    ``mirror_machine_config_to_file`` JSON-encodes ``dict`` / ``list`` values
+    so they round-trip through INI's string-only storage. When reading the
+    file back into ``Machine.config`` (a JSONField that holds native types)
+    those strings have to be decoded — otherwise downstream consumers like
+    ``_emit_machine_config`` → ``MachineEvent`` → abx-dl see a JSON string
+    where they expect a dict and raise ``TypeError``.
+    Declared fields go through pydantic-settings' own ``field_is_complex`` /
+    ``prepare_field_value`` so they're decoded per annotation. Undeclared
+    keys (e.g. ``ABX_INSTALL_CACHE``, written dynamically by abx-dl) are
+    JSON-decoded when their string starts with ``{`` or ``[`` — the same
+    shape ``_coerce_to_str_dict`` writes them as.
+    """
+    from archivebox.config.common import ArchiveBoxConfig
+    from archivebox.config.configset import IniConfigSettingsSource
+
+    decoder = IniConfigSettingsSource(ArchiveBoxConfig)
+    decoded: dict[str, Any] = dict(file_config)
+    declared_fields = set(ArchiveBoxConfig.model_fields)
+    for field_name, field in ArchiveBoxConfig.model_fields.items():
+        if field_name not in decoded:
+            continue
+        raw = decoded[field_name]
+        if not isinstance(raw, str) or not raw:
+            continue
+        if decoder.field_is_complex(field):
+            decoded[field_name] = decoder.prepare_field_value(field_name, field, raw, True)
+    for key, raw in list(decoded.items()):
+        if key in declared_fields:
+            continue
+        if not isinstance(raw, str) or not raw:
+            continue
+        first = raw[:1]
+        if first not in ("{", "["):
+            continue
+        try:
+            decoded[key] = json.loads(raw)
+        except (TypeError, ValueError):
+            continue
+    return decoded
+
+
+def _mirror_file_to_machine_config(file_config: dict[str, str]) -> None:
+    """Copy ``ArchiveBox.conf`` contents into ``Machine.config``.
+
+    Internal helper used by ``write_config_file`` and the startup sync —
+    callers must hold the ``_MIRROR_IN_PROGRESS`` guard around it.
+    """
+    from archivebox.machine.models import Machine
+
+    machine = Machine.current()
+    if _coerce_to_str_dict(machine.config) == file_config:
+        return
+    machine.config = _coerce_from_str_dict(file_config)
+    machine.save(update_fields=["config", "modified_at"])
+
+
+def sync_machine_and_file(machine: Any = None) -> None:
+    """One-time-per-process reconciliation between the two stores.
+
+    Cheap on the common case where they already agree (single ``stat`` + dict
+    compare ≈ 1ms). When the two sides diverge we merge them: each side's
+    unique keys are preserved, and for keys present on both we let the newer
+    side win (file mtime vs. ``Machine.modified_at``). After the merge both
+    stores hold the union, so every subsequent write keeps them in lockstep
+    via the full-replace mirror functions.
+
+    Pass ``machine`` when the caller already has a current ``Machine``
+    instance (e.g. from ``Machine.current()``) to skip the 10–15ms
+    ``get_host_guid()`` round-trip on the cold path.
+    """
+    global _INITIAL_SYNC_DONE, _MIRROR_IN_PROGRESS
+    if _INITIAL_SYNC_DONE:
+        return
+    _INITIAL_SYNC_DONE = True
+    if _MIRROR_IN_PROGRESS:
+        return
+    _MIRROR_IN_PROGRESS = True
+    try:
+        if machine is None:
+            from archivebox.machine.detect import get_host_guid
+            from archivebox.machine.models import Machine
+
+            try:
+                machine = Machine.objects.filter(guid=get_host_guid()).first()
+            except Exception:
+                return
+            if machine is None:
+                return
+
+        file_config, file_mtime = _load_file_config_dict()
+        machine_config = _coerce_to_str_dict(machine.config)
+        if machine_config == file_config:
+            return
+
+        db_mtime = machine.modified_at.timestamp() if machine.modified_at else 0.0
+        file_is_newer = file_mtime is not None and file_mtime > db_mtime
+
+        merged: dict[str, str] = {}
+        all_keys = set(machine_config) | set(file_config)
+        for key in all_keys:
+            in_file = key in file_config
+            in_db = key in machine_config
+            if in_file and in_db:
+                if file_config[key] == machine_config[key]:
+                    merged[key] = file_config[key]
+                else:
+                    merged[key] = file_config[key] if file_is_newer else machine_config[key]
+            elif in_file:
+                merged[key] = file_config[key]
+            else:
+                merged[key] = machine_config[key]
+
+        if merged != file_config:
+            _write_file_if_changed(_render_config_file_content(merged))
+        if merged != machine_config:
+            machine.config = _coerce_from_str_dict(merged)
+            machine.save(update_fields=["config", "modified_at"])
+    finally:
+        _MIRROR_IN_PROGRESS = False
+
+
+def write_config_file(config: dict[str, str]) -> AttrDict:
+    """Merge ``config`` into ``ArchiveBox.conf``, validate, then mirror to Machine.config.
+
+    Backwards-compatible signature: callers (CLI ``archivebox config --set``
+    and the init flow) pass a partial dict of keys to upsert.
+    """
+
+    from archivebox.config.common import get_all_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
+    from archivebox.misc.system import atomic_write
 
     config_path = CONSTANTS.CONFIG_FILE
 
     if not os.access(config_path, os.F_OK):
-        atomic_write(config_path, CONFIG_HEADER)
+        atomic_write(config_path, CONFIG_FILE_HEADER)
 
     config_file = CaseConfigParser()
     config_file.read(config_path)
@@ -44,27 +304,13 @@ def write_config_file(config: dict[str, str]) -> benedict:
 
     # Set up sections in empty config file
     for key, val in config.items():
-        section_name = None
-        for section in config_sections.values():
-            if key in type(section).model_fields:
-                section_name = section.toml_section_header
-                break
-
-        if section_name is None:
-            for schema in plugin_configs.values():
-                if "properties" in schema and key in schema["properties"]:
-                    section_name = "PLUGINS"
-                    break
-
-        if section_name is None:
-            raise ValueError(f"No config section found for key: {key}")
-
+        section_name = _resolve_section_for_key(key, config_sections, plugin_configs)
         if section_name in config_file:
             existing_config = dict(config_file[section_name])
         else:
             existing_config = {}
 
-        config_file[section_name] = benedict({**existing_config, key: val})
+        config_file[section_name] = AttrDict({**existing_config, key: val})
 
     with open(config_path, "w+", encoding="utf-8") as new:
         config_file.write(new)
@@ -85,4 +331,18 @@ def write_config_file(config: dict[str, str]) -> benedict:
     if os.access(f"{config_path}.bak", os.F_OK):
         os.remove(f"{config_path}.bak")
 
-    return benedict({key.upper(): updated_config.get(key.upper()) for key in config.keys()})
+    # Mirror the post-write file state into Machine.config so the DB stays
+    # 1:1 with the on-disk file. Recursion-guarded so Machine.save's own
+    # mirror-back doesn't loop us.
+    global _MIRROR_IN_PROGRESS
+    if not _MIRROR_IN_PROGRESS:
+        _MIRROR_IN_PROGRESS = True
+        try:
+            flat, _mtime = _load_file_config_dict()
+            _mirror_file_to_machine_config(flat)
+        except Exception:
+            pass
+        finally:
+            _MIRROR_IN_PROGRESS = False
+
+    return AttrDict({key.upper(): updated_config.get(key.upper()) for key in config.keys()})

archivebox/config/configset.py

@@ -22,8 +22,6 @@ COMPUTED_CONFIG_KEYS = (
     "IS_LOWER_SECURITY_MODE",
     "URL_ALLOWLIST_PTN",
     "URL_DENYLIST_PTN",
-    "SAVE_ALLOWLIST_PTNS",
-    "SAVE_DENYLIST_PTNS",
 )
 
 
@@ -32,6 +30,33 @@ class CaseConfigParser(ConfigParser):
         return optionstr
 
 
+# ``IniConfigSettingsSource.get_field_value`` is called once per pydantic field
+# (hundreds of fields across the GlobalConfig + plugin configs). Without
+# caching, every field re-opens and re-parses ``ArchiveBox.conf`` from disk —
+# that's the dominant cost of ``get_config()`` (each call was ~130ms; profile
+# showed 632 ``parser.read(config_path)`` invocations per call). The cache is
+# keyed on (path, mtime) so external edits to the file still get picked up.
+_INI_CACHE: dict[tuple[str, float], dict[str, Any]] = {}
+
+
+def _read_ini_config_cached(config_path_str: str) -> dict[str, Any]:
+    config_path = Path(config_path_str)
+    try:
+        mtime = config_path.stat().st_mtime
+    except FileNotFoundError:
+        return {}
+    cache_key = (str(config_path), mtime)
+    cached = _INI_CACHE.get(cache_key)
+    if cached is not None:
+        return cached
+    parser = CaseConfigParser()
+    parser.read(config_path)
+    flat = {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+    _INI_CACHE.clear()
+    _INI_CACHE[cache_key] = flat
+    return flat
+
+
 class IniConfigSettingsSource(PydanticBaseSettingsSource):
     """
     Custom settings source that reads from ArchiveBox.conf (INI format).
@@ -41,10 +66,27 @@ class IniConfigSettingsSource(PydanticBaseSettingsSource):
     def get_field_value(self, field: Any, field_name: str) -> tuple[Any, str, bool]:
         config_vals = self._load_config_file()
         field_value = config_vals.get(field_name.upper())
-        return field_value, field_name, False
+        # Mark complex-typed fields (``dict``/``list``) so the parent
+        # ``prepare_field_value`` JSON-decodes the INI string before pydantic
+        # validates against the dict/list annotation. Without this, e.g.
+        # ``ABX_INSTALL_CACHE`` arrives as a raw JSON string and pydantic
+        # rejects it with ``Input should be a valid dictionary``.
+        value_is_complex = bool(field_value is not None and self.field_is_complex(field))
+        return field_value, field_name, value_is_complex
 
     def __call__(self) -> dict[str, Any]:
-        return self._load_config_file()
+        # Use the per-field path (``get_field_value`` + ``prepare_field_value``)
+        # so complex types get JSON-decoded. The previous flat-dict return
+        # skipped pydantic-settings' complex-value handling entirely.
+        result: dict[str, Any] = {}
+        for field_name, field in self.settings_cls.model_fields.items():
+            value, key, value_is_complex = self.get_field_value(field, field_name)
+            if value is None:
+                continue
+            prepared = self.prepare_field_value(field_name, field, value, value_is_complex)
+            if prepared is not None:
+                result[key] = prepared
+        return result
 
     def _load_config_file(self) -> dict[str, Any]:
         try:
@@ -54,14 +96,7 @@ class IniConfigSettingsSource(PydanticBaseSettingsSource):
         except ImportError:
             return {}
 
-        if not config_path.exists():
-            return {}
-
-        parser = CaseConfigParser()
-        parser.read(config_path)
-
-        # Flatten all sections into single namespace (ignore section headers)
-        return {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+        return _read_ini_config_cached(str(config_path))
 
 
 class BaseConfigSet(BaseSettings):

archivebox/config/constants.py

@@ -17,10 +17,9 @@ import sys
 
 from pathlib import Path
 
-from benedict import benedict
 from platformdirs import user_config_path
 
-from archivebox.misc.logging import DEFAULT_CLI_COLORS
+from archivebox.misc.logging import AttrDict, DEFAULT_CLI_COLORS
 
 from .paths import (
     PACKAGE_DIR,
@@ -37,8 +36,8 @@ from .permissions import (
     IN_DOCKER,
     RUNNING_AS_UID,
     RUNNING_AS_GID,
-    DEFAULT_PUID,
-    DEFAULT_PGID,
+    DEFAULT_UID,
+    DEFAULT_GID,
     ARCHIVEBOX_USER,
     ARCHIVEBOX_GROUP,
 )
@@ -67,8 +66,8 @@ class ConstantsDict:
     ARCHIVEBOX_GROUP: int = ARCHIVEBOX_GROUP
     RUNNING_AS_UID: int = RUNNING_AS_UID
     RUNNING_AS_GID: int = RUNNING_AS_GID
-    DEFAULT_PUID: int = DEFAULT_PUID
-    DEFAULT_PGID: int = DEFAULT_PGID
+    DEFAULT_UID: int = DEFAULT_UID
+    DEFAULT_GID: int = DEFAULT_GID
     IS_INSIDE_VENV: bool = sys.prefix != sys.base_prefix
 
     # Source code dirs
@@ -114,12 +113,10 @@ class ConstantsDict:
     TMP_DIR_NAME: str = "tmp"
     DEFAULT_TMP_DIR: Path = DATA_DIR / TMP_DIR_NAME / MACHINE_ID  # ./data/tmp/abc3244323
 
-    LIB_DIR_NAME: str = "lib"
-    DEFAULT_LIB_DIR: Path = _env_path(
-        "LIB_DIR",
-        user_config_path("abx") / LIB_DIR_NAME,
+    DEFAULT_ABXPKG_LIB_DIR: Path = _env_path(
+        "ABXPKG_LIB_DIR",
+        user_config_path("abx") / "lib",
     )
-    DEFAULT_LIB_BIN_DIR: Path = _env_path("LIB_BIN_DIR", DEFAULT_LIB_DIR / "bin")
 
     RESERVED_ARCHIVE_DIR_NAMES: frozenset[str] = frozenset(
         (
@@ -134,7 +131,7 @@ class ConstantsDict:
     # Config constants
     TIMEZONE: str = "UTC"
     DEFAULT_CLI_COLORS: dict[str, str] = DEFAULT_CLI_COLORS
-    DISABLED_CLI_COLORS: dict[str, str] = benedict({k: "" for k in DEFAULT_CLI_COLORS})
+    DISABLED_CLI_COLORS: dict[str, str] = AttrDict({k: "" for k in DEFAULT_CLI_COLORS})
 
     # Hard safety limits (seconds)
     MAX_HOOK_RUNTIME_SECONDS: int = 60 * 60 * 12  # 12 hours
@@ -242,7 +239,7 @@ class ConstantsDict:
             SOURCES_DIR_NAME,
             LOGS_DIR_NAME,
             CACHE_DIR_NAME,
-            LIB_DIR_NAME,
+            "lib",
             TMP_DIR_NAME,
             PERSONAS_DIR_NAME,
             CUSTOM_TEMPLATES_DIR_NAME,
@@ -291,14 +288,9 @@ class ConstantsDict:
         # so it behaves like a dict[key] == dict.key or object attr
         return getattr(cls, key)
 
-    @classmethod
-    def __benedict__(cls):
-        # when casting to benedict, only include uppercase keys that don't start with an underscore
-        return benedict({key: value for key, value in cls.__dict__.items() if key.isupper() and not key.startswith("_")})
-
 
 CONSTANTS = ConstantsDict
-CONSTANTS_CONFIG = CONSTANTS.__benedict__()
+CONSTANTS_CONFIG = AttrDict({key: value for key, value in CONSTANTS.__dict__.items() if key.isupper() and not key.startswith("_")})
 
 # add all key: values to globals() for easier importing, e.g.:
 # from archivebox.config.constants import IS_ROOT, PERSONAS_DIR, ...

archivebox/config/django.py

@@ -28,14 +28,6 @@ STDERR = Console(stderr=True)
 logging.CONSOLE = CONSOLE
 
 
-def setup_django_minimal():
-    # sys.path.append(str(CONSTANTS.PACKAGE_DIR))
-    # os.environ.setdefault('ARCHIVEBOX_DATA_DIR', str(CONSTANTS.DATA_DIR))
-    # os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
-    # django.setup()
-    raise Exception("dont use this anymore")
-
-
 DJANGO_SET_UP = False
 
 
@@ -49,6 +41,17 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
         # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
         return
 
+    # SQLite creates index.sqlite3 during django.setup()/migrate. Apply the
+    # ArchiveBox file-mode policy before any DB connection can create the file,
+    # otherwise a permissive parent umask can expose a just-created DB until a
+    # later chmod runs.
+    os.umask(0o777 - (int(CONFIG.OUTPUT_PERMISSIONS, base=8) | 0o111))
+
+    # Third-party patches are only needed once Django/apps are about to load.
+    # Keeping them out of archivebox.__init__ avoids paying Django/Daphne setup
+    # cost for cheap CLI startup paths like `archivebox <cmd> --help`.
+    import archivebox.misc.monkey_patches  # noqa: F401
+
     from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, SudoPermission
 
     # if running as root, chown the data dir to the archivebox user to make sure it's accessible to the archivebox user
@@ -165,6 +168,6 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
             #     logfire.info(f'Started ArchiveBox v{CONSTANTS.VERSION}', argv=sys.argv)
 
     except KeyboardInterrupt:
-        raise SystemExit(2)
+        raise
 
     DJANGO_SET_UP = True

archivebox/config/ldap.py

@@ -1,6 +1,6 @@
 __package__ = "archivebox.config"
 
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 
 from archivebox.config.configset import BaseConfigSet
 
@@ -14,6 +14,7 @@ class LDAPConfig(BaseConfigSet):
     """
 
     toml_section_header: str = "LDAP_CONFIG"
+    _scope: str = PrivateAttr(default="server")
 
     LDAP_ENABLED: bool = Field(default=False)
     LDAP_SERVER_URI: str | None = Field(default=None)

archivebox/config/paths.py

@@ -11,8 +11,6 @@ from functools import cache
 from datetime import datetime
 from typing import TYPE_CHECKING
 
-from benedict import benedict
-
 from .permissions import SudoPermission, IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
 
 if TYPE_CHECKING:
@@ -21,7 +19,9 @@ if TYPE_CHECKING:
 #############################################################################################
 
 PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent  # archivebox source code dir
-DATA_DIR: Path = Path(os.environ.get("DATA_DIR", os.getcwd())).resolve()  # archivebox user data dir
+DATA_DIR: Path = Path(os.getcwd()).resolve()  # archivebox user data dir
+MAX_TMP_SOCKET_URL_LENGTH = 90
+SUPERVISORD_SOCKET_FILENAME = "supervisord.sock"
 
 
 def _env_path(key: str, default: Path) -> Path:
@@ -31,8 +31,8 @@ def _env_path(key: str, default: Path) -> Path:
     return path.resolve()
 
 
-ARCHIVE_DIR: Path = _env_path("ARCHIVE_DIR", DATA_DIR / "archive")  # archivebox snapshot data dir
-USERS_DIR: Path = _env_path("USERS_DIR", ARCHIVE_DIR / "users")  # archivebox user-scoped crawl/snapshot data dir
+ARCHIVE_DIR: Path = DATA_DIR / "archive"  # archivebox snapshot data dir
+USERS_DIR: Path = ARCHIVE_DIR / "users"  # archivebox user-scoped crawl/snapshot data dir
 
 IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
 
@@ -122,7 +122,7 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No
     try:
         with SudoPermission(uid=uid, fallback=fallback):
             test_file.exists()
-            test_file.write_text(f"Checking if PUID={uid} PGID={gid} can write to dir")
+            test_file.write_text(f"Checking if uid={uid} gid={gid} can write to dir")
             test_file.unlink()
             return True
     except (OSError, PermissionError):
@@ -178,33 +178,45 @@ def create_and_chown_dir(dir_path: Path) -> None:
 
 
 def tmp_dir_socket_path_is_short_enough(dir_path: Path) -> bool:
-    socket_file = dir_path.absolute().resolve() / "supervisord.sock"
-    return len(f"file://{socket_file}") <= 96
+    socket_file = dir_path.absolute().resolve() / SUPERVISORD_SOCKET_FILENAME
+    return len(f"file://{socket_file}") < MAX_TMP_SOCKET_URL_LENGTH
+
+
+def tmp_dir_candidates(config: "ArchiveBoxConfig") -> list[Path]:
+    from archivebox.config.constants import CONSTANTS
+
+    collection_id = get_collection_id()
+    collection_id_short = collection_id[:4]
+    system_tmp_dir = Path(tempfile.gettempdir())
+    candidates = [
+        config.TMP_DIR,  # <user-specified>
+        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
+        Path("/var/run/archivebox") / collection_id,
+        Path("/tmp") / "archivebox" / collection_id,
+        Path("~/.tmp/archivebox").expanduser() / collection_id,
+        system_tmp_dir / "archivebox" / collection_id,
+        system_tmp_dir / "archivebox" / collection_id_short,
+        system_tmp_dir / "abx" / collection_id_short,
+    ]
+    seen = set()
+    unique_candidates = []
+    for path in candidates:
+        path_key = str(path.expanduser().absolute())
+        if path_key in seen:
+            continue
+        seen.add(path_key)
+        unique_candidates.append(path)
+    return unique_candidates
 
 
 def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
-    from archivebox.config.constants import CONSTANTS
     from archivebox.config.common import get_config
     from archivebox.misc.checks import check_tmp_dir
 
     config = config or get_config(**config_kwargs)
-    # try a few potential directories in order of preference
-    CANDIDATES = [
-        config.TMP_DIR,  # <user-specified>
-        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
-        Path("/var/run/archivebox") / get_collection_id(),  # /var/run/archivebox/abc5d8512
-        Path("/tmp") / "archivebox" / get_collection_id(),  # /tmp/archivebox/abc5d8512
-        Path("~/.tmp/archivebox").expanduser() / get_collection_id(),  # ~/.tmp/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id(),  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d
-        Path(tempfile.gettempdir()) / "abx" / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/abx/abc5
-    ]
+    candidates = tmp_dir_candidates(config)
     fallback_candidate = None
-    for candidate in CANDIDATES:
+    for candidate in candidates:
         try:
             create_and_chown_dir(candidate)
         except Exception:
@@ -233,7 +245,7 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxC
         return fallback_candidate
 
     if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {CANDIDATES}!")
+        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {candidates}!")
 
 
 def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@@ -242,8 +254,8 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
 
     config = config or get_config(**config_kwargs)
 
-    # LIB_DIR is either the shared platformdirs default or an explicit env/config override.
-    CANDIDATES = [config.LIB_DIR]
+    # ABXPKG_LIB_DIR is either the shared platformdirs default or an explicit env/config override.
+    CANDIDATES = [config.ABXPKG_LIB_DIR]
 
     for candidate in CANDIDATES:
         try:
@@ -251,17 +263,18 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
         except Exception:
             pass
         if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True):
-            if autofix and config.LIB_DIR != candidate:
-                os.environ["LIB_DIR"] = str(candidate)
+            if autofix and config.ABXPKG_LIB_DIR != candidate:
+                os.environ["ABXPKG_LIB_DIR"] = str(candidate)
             return candidate
 
     if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable LIB_DIR, tried {CANDIDATES}!")
+        raise OSError(f"ArchiveBox is unable to find a writable ABXPKG_LIB_DIR, tried {CANDIDATES}!")
 
 
 def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs):
     from archivebox.config.constants import CONSTANTS
     from archivebox.config.common import get_config
+    from archivebox.misc.logging import AttrDict
 
     config = config or get_config(**config_kwargs)
     try:
@@ -269,7 +282,7 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
     except Exception:
         tmp_dir = config.TMP_DIR
 
-    return benedict(
+    return AttrDict(
         {
             "DATA_DIR": {
                 "path": DATA_DIR.resolve(),
@@ -291,20 +304,20 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
             },
             "ARCHIVE_DIR": {
-                "path": config.ARCHIVE_DIR.resolve(),
+                "path": CONSTANTS.ARCHIVE_DIR.resolve(),
                 "enabled": True,
-                "is_valid": os.path.isdir(config.ARCHIVE_DIR)
-                and os.access(config.ARCHIVE_DIR, os.R_OK)
-                and os.access(config.ARCHIVE_DIR, os.W_OK),
-                "is_mount": os.path.ismount(config.ARCHIVE_DIR.resolve()),
+                "is_valid": os.path.isdir(CONSTANTS.ARCHIVE_DIR)
+                and os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK)
+                and os.access(CONSTANTS.ARCHIVE_DIR, os.W_OK),
+                "is_mount": os.path.ismount(CONSTANTS.ARCHIVE_DIR.resolve()),
             },
             "USERS_DIR": {
-                "path": config.USERS_DIR.resolve(),
-                "enabled": os.path.isdir(config.USERS_DIR),
-                "is_valid": os.path.isdir(config.USERS_DIR)
-                and os.access(config.USERS_DIR, os.R_OK)
-                and os.access(config.USERS_DIR, os.W_OK),
-                "is_mount": os.path.ismount(config.USERS_DIR.resolve()),
+                "path": CONSTANTS.USERS_DIR.resolve(),
+                "enabled": os.path.isdir(CONSTANTS.USERS_DIR),
+                "is_valid": os.path.isdir(CONSTANTS.USERS_DIR)
+                and os.access(CONSTANTS.USERS_DIR, os.R_OK)
+                and os.access(CONSTANTS.USERS_DIR, os.W_OK),
+                "is_mount": os.path.ismount(CONSTANTS.USERS_DIR.resolve()),
             },
             "SOURCES_DIR": {
                 "path": CONSTANTS.SOURCES_DIR.resolve(),
@@ -330,7 +343,10 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
             "TMP_DIR": {
                 "path": tmp_dir.resolve(),
                 "enabled": True,
-                "is_valid": os.path.isdir(tmp_dir) and os.access(tmp_dir, os.R_OK) and os.access(tmp_dir, os.W_OK),  # read + write
+                "is_valid": os.path.isdir(tmp_dir)
+                and os.access(tmp_dir, os.R_OK)
+                and os.access(tmp_dir, os.W_OK)
+                and tmp_dir_socket_path_is_short_enough(tmp_dir),
             },
             # "CACHE_DIR": {
             #     "path": CACHE_DIR.resolve(),
@@ -344,16 +360,15 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
 def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs):
     from archivebox.config.constants import CONSTANTS
     from archivebox.config.common import get_config
+    from archivebox.misc.logging import AttrDict
 
     config = config or get_config(**config_kwargs)
     try:
-        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR
+        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.ABXPKG_LIB_DIR
     except Exception:
-        lib_dir = config.LIB_DIR
+        lib_dir = config.ABXPKG_LIB_DIR
 
-    lib_bin_dir = lib_dir / "bin"
-
-    return benedict(
+    return AttrDict(
         {
             "PACKAGE_DIR": {
                 "path": (PACKAGE_DIR).resolve(),
@@ -366,26 +381,19 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 "is_valid": os.access(CONSTANTS.STATIC_DIR, os.R_OK) and os.access(CONSTANTS.STATIC_DIR, os.X_OK),  # read + list
             },
             "CUSTOM_TEMPLATES_DIR": {
-                "path": config.CUSTOM_TEMPLATES_DIR.resolve(),
-                "enabled": os.path.isdir(config.CUSTOM_TEMPLATES_DIR),
-                "is_valid": os.path.isdir(config.CUSTOM_TEMPLATES_DIR) and os.access(config.CUSTOM_TEMPLATES_DIR, os.R_OK),  # read
+                "path": CONSTANTS.CUSTOM_TEMPLATES_DIR.resolve(),
+                "enabled": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR),
+                "is_valid": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK),  # read
             },
             "USER_PLUGINS_DIR": {
                 "path": CONSTANTS.USER_PLUGINS_DIR.resolve(),
                 "enabled": os.path.isdir(CONSTANTS.USER_PLUGINS_DIR),
                 "is_valid": os.path.isdir(CONSTANTS.USER_PLUGINS_DIR) and os.access(CONSTANTS.USER_PLUGINS_DIR, os.R_OK),  # read
             },
-            "LIB_DIR": {
+            "ABXPKG_LIB_DIR": {
                 "path": lib_dir.resolve(),
                 "enabled": True,
                 "is_valid": os.path.isdir(lib_dir) and os.access(lib_dir, os.R_OK) and os.access(lib_dir, os.W_OK),  # read + write
             },
-            "LIB_BIN_DIR": {
-                "path": lib_bin_dir.resolve(),
-                "enabled": True,
-                "is_valid": os.path.isdir(lib_bin_dir)
-                and os.access(lib_bin_dir, os.R_OK)
-                and os.access(lib_bin_dir, os.W_OK),  # read + write
-            },
         },
     )

archivebox/config/permissions.py

@@ -24,8 +24,8 @@ except PermissionError:
     DATA_DIR_UID = 0
     DATA_DIR_GID = 0
 
-DEFAULT_PUID = 911
-DEFAULT_PGID = 911
+DEFAULT_UID = 911
+DEFAULT_GID = 911
 RUNNING_AS_UID = os.getuid()
 RUNNING_AS_GID = os.getgid()
 EUID = os.geteuid()
@@ -42,23 +42,20 @@ IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE",
 
 FALLBACK_UID = RUNNING_AS_UID or SUDO_UID
 FALLBACK_GID = RUNNING_AS_GID or SUDO_GID
-if RUNNING_AS_UID == 0:
-    try:
-        # if we are running as root it's really hard to figure out what the correct archivebox user should be
-        # as a last resort instead of setting DATA_DIR ownership to 0:0 (which breaks it for non-root users)
-        # check if 911:911 archivebox user exists on host system, and use it instead of 0
-        if pwd.getpwuid(DEFAULT_PUID).pw_name == "archivebox":
-            FALLBACK_UID = DEFAULT_PUID
-            FALLBACK_GID = DEFAULT_PGID
-    except Exception:
-        pass
+try:
+    ARCHIVEBOX_ACCOUNT = pwd.getpwnam("archivebox")
+except KeyError:
+    ARCHIVEBOX_ACCOUNT = None
 
-
-os.environ.setdefault("PUID", str(DATA_DIR_UID or EUID or RUNNING_AS_UID or FALLBACK_UID))
-os.environ.setdefault("PGID", str(DATA_DIR_GID or EGID or RUNNING_AS_GID or FALLBACK_GID))
-
-ARCHIVEBOX_USER = int(os.environ["PUID"])
-ARCHIVEBOX_GROUP = int(os.environ["PGID"])
+if DATA_DIR_UID != 0:
+    ARCHIVEBOX_USER = DATA_DIR_UID
+    ARCHIVEBOX_GROUP = DATA_DIR_GID
+elif RUNNING_AS_UID == 0 and ARCHIVEBOX_ACCOUNT is not None:
+    ARCHIVEBOX_USER = ARCHIVEBOX_ACCOUNT.pw_uid
+    ARCHIVEBOX_GROUP = ARCHIVEBOX_ACCOUNT.pw_gid
+else:
+    ARCHIVEBOX_USER = EUID or RUNNING_AS_UID or FALLBACK_UID
+    ARCHIVEBOX_GROUP = EGID or RUNNING_AS_GID or FALLBACK_GID
 if not USER:
     try:
         # alternative method 1 to get username
@@ -94,13 +91,12 @@ except Exception:
 
 
 def drop_privileges():
-    """If running as root, drop privileges to the user that owns the data dir (or PUID)"""
+    """If running as root, drop privileges to the data dir owner or archivebox user."""
 
-    # always run archivebox as the user that owns the data dir, never as root
+    # Always run ArchiveBox as the user that owns the data dir, or as the
+    # archivebox service account when the data dir is root-owned.
     if os.getuid() == 0:
-        # drop permissions to the user that owns the data dir / provided PUID
         if os.geteuid() != ARCHIVEBOX_USER and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
-            # drop our effective UID to the archivebox user's UID
             os.seteuid(ARCHIVEBOX_USER)
 
             # update environment variables so that subprocesses dont try to write to /root

archivebox/config/version.py

@@ -6,6 +6,7 @@ import importlib.metadata
 from pathlib import Path
 from functools import cache
 from datetime import datetime
+import re
 
 #############################################################################################
 
@@ -41,11 +42,57 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR):
 
 @cache
 def get_COMMIT_HASH() -> str | None:
+    for env_var in ("ARCHIVEBOX_COMMIT_HASH", "COMMIT_HASH"):
+        env_commit_hash = os.environ.get(env_var, "").strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", env_commit_hash):
+            return env_commit_hash
+
+    if IN_DOCKER:
+        try:
+            version_text = Path("/VERSION.txt").read_text()
+            matches = re.findall(r"^COMMIT_HASH=([0-9a-fA-F]{40})$", version_text, re.MULTILINE)
+            if matches:
+                return matches[-1]
+        except Exception:
+            pass
+
+    def _read_git_file(git_dir: Path, ref: str) -> str | None:
+        try:
+            return git_dir.joinpath(ref).read_text().strip()
+        except Exception:
+            pass
+
+        try:
+            packed_refs = git_dir.joinpath("packed-refs").read_text().splitlines()
+        except Exception:
+            return None
+
+        for line in packed_refs:
+            if line.startswith("#") or line.startswith("^") or not line.strip():
+                continue
+            commit_hash, packed_ref = line.split(" ", 1)
+            if packed_ref == ref:
+                return commit_hash.strip()
+
+        return None
+
     try:
         git_dir = PACKAGE_DIR.parent / ".git"
-        ref = (git_dir / "HEAD").read_text().strip().split(" ")[-1]
-        commit_hash = git_dir.joinpath(ref).read_text().strip()
-        return commit_hash
+        if git_dir.is_file():
+            gitdir_line = git_dir.read_text().strip()
+            gitdir_path = gitdir_line.removeprefix("gitdir:").strip()
+            git_dir = Path(gitdir_path)
+            if not git_dir.is_absolute():
+                git_dir = PACKAGE_DIR.parent / git_dir
+
+        head = (git_dir / "HEAD").read_text().strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", head):
+            return head
+
+        ref = head.removeprefix("ref:").strip()
+        commit_hash = _read_git_file(git_dir, ref)
+        if commit_hash:
+            return commit_hash
     except Exception:
         pass
 
@@ -59,13 +106,10 @@ def get_COMMIT_HASH() -> str | None:
 
 @cache
 def get_BUILD_TIME() -> str:
-    if IN_DOCKER:
-        try:
-            # if we're in the archivebox official docker image, /VERSION.txt will contain the build time
-            docker_build_end_time = Path("/VERSION.txt").read_text().rsplit("BUILD_END_TIME=")[-1].split("\n", 1)[0]
-            return docker_build_end_time
-        except Exception:
-            pass
+    for env_var in ("ARCHIVEBOX_BUILD_TIME", "BUILD_TIME"):
+        build_time = os.environ.get(env_var, "").strip()
+        if build_time:
+            return build_time
 
     src_last_modified_unix_timestamp = (PACKAGE_DIR / "README.md").stat().st_mtime
     return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime("%Y-%m-%d %H:%M:%S %s")

archivebox/config/views.py

@@ -1,13 +1,9 @@
 __package__ = "archivebox.config"
 
-import html
-import json
 import os
 import inspect
-import re
 from pathlib import Path
 from typing import Any
-from collections.abc import Callable
 from urllib.parse import quote, urlencode
 from django.http import HttpRequest
 from django.utils import timezone
@@ -22,15 +18,12 @@ from archivebox.misc.util import parse_date
 
 from archivebox.machine.models import Binary
 
-ABX_PLUGINS_DOCS_BASE_URL = "https://archivebox.github.io/abx-plugins/"
-ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/"
-LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/"
 INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"
 
 
 def is_superuser(request: HttpRequest) -> bool:
-    return bool(getattr(request.user, "is_superuser", False))
+    return bool(request.user.is_superuser)
 
 
 def format_parsed_datetime(value: object) -> str:
@@ -38,59 +31,6 @@ def format_parsed_datetime(value: object) -> str:
     return parsed.strftime("%Y-%m-%d %H:%M:%S") if parsed else ""
 
 
-JSON_TOKEN_RE = re.compile(
-    r'(?P<key>"(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*")(?=\s*:)'
-    r'|(?P<string>"(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*")'
-    r"|(?P<boolean>\btrue\b|\bfalse\b)"
-    r"|(?P<null>\bnull\b)"
-    r"|(?P<number>-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)",
-)
-
-
-def render_code_block(text: str, *, highlighted: bool = False) -> str:
-    code = html.escape(text, quote=False)
-
-    if highlighted:
-
-        def _wrap_token(match: re.Match[str]) -> str:
-            styles = {
-                "key": "color: #0550ae;",
-                "string": "color: #0a7f45;",
-                "boolean": "color: #8250df; font-weight: 600;",
-                "null": "color: #6e7781; font-style: italic;",
-                "number": "color: #b35900;",
-            }
-            token_type = next(name for name, value in match.groupdict().items() if value is not None)
-            return f'<span style="{styles[token_type]}">{match.group(0)}</span>'
-
-        code = JSON_TOKEN_RE.sub(_wrap_token, code)
-
-    return (
-        '<pre style="max-height: 600px; overflow: auto; background: #f6f8fa; '
-        'border: 1px solid #d0d7de; border-radius: 6px; padding: 12px; margin: 0;">'
-        '<code style="font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, '
-        "'Liberation Mono', monospace; white-space: pre; line-height: 1.5;\">"
-        f"{code}"
-        "</code></pre>"
-    )
-
-
-def render_highlighted_json_block(value: Any) -> str:
-    return render_code_block(json.dumps(value, indent=2, ensure_ascii=False), highlighted=True)
-
-
-def get_plugin_docs_url(plugin_name: str) -> str:
-    return f"{ABX_PLUGINS_DOCS_BASE_URL}#{plugin_name}"
-
-
-def get_plugin_hook_source_url(plugin_name: str, hook_name: str) -> str:
-    return f"{ABX_PLUGINS_GITHUB_BASE_URL}{quote(plugin_name)}/{quote(hook_name)}"
-
-
-def get_live_config_url(key: str) -> str:
-    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
-
-
 def get_environment_binary_url(name: str) -> str:
     return f"{ENVIRONMENT_BINARIES_BASE_URL}{quote(name)}/"
 
@@ -99,199 +39,11 @@ def get_installed_binary_change_url(name: str, binary: Binary | None) -> str | N
     if binary is None or not binary.id:
         return None
 
-    base_url = getattr(binary, "admin_change_url", None) or f"{INSTALLED_BINARIES_BASE_URL}{binary.id}/change/"
+    base_url = binary.admin_change_url
     changelist_filters = urlencode({"q": name})
     return f"{base_url}?{urlencode({'_changelist_filters': changelist_filters})}"
 
 
-def get_machine_admin_url() -> str | None:
-    try:
-        from archivebox.machine.models import Machine
-
-        return Machine.current().admin_change_url
-    except Exception:
-        return None
-
-
-def render_code_tag_list(values: list[str]) -> str:
-    if not values:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    tags = "".join(
-        str(
-            format_html(
-                '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>',
-                value,
-            ),
-        )
-        for value in values
-    )
-    return f'<div style="display: flex; flex-wrap: wrap;">{tags}</div>'
-
-
-def render_plugin_metadata_html(config: dict[str, Any]) -> str:
-    required_binaries = [
-        str(item.get("name")) for item in (config.get("required_binaries") or []) if isinstance(item, dict) and item.get("name")
-    ]
-    rows = (
-        ("Title", config.get("title") or "(none)"),
-        ("Description", config.get("description") or "(none)"),
-        ("Required Plugins", mark_safe(render_link_tag_list(config.get("required_plugins") or [], get_plugin_docs_url))),
-        ("Required Binaries", mark_safe(render_link_tag_list(required_binaries, get_environment_binary_url))),
-        ("Output MIME Types", mark_safe(render_code_tag_list(config.get("output_mimetypes") or []))),
-    )
-
-    rendered_rows = "".join(
-        str(
-            format_html(
-                '<div style="margin: 0 0 14px 0;"><div style="font-weight: 600; margin-bottom: 4px;">{}</div><div>{}</div></div>',
-                label,
-                value,
-            ),
-        )
-        for label, value in rows
-    )
-    return f'<div style="margin: 4px 0 0 0;">{rendered_rows}</div>'
-
-
-def render_link_tag_list(values: list[str], url_resolver: Callable[[str], str] | None = None) -> str:
-    if not values:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    tags = []
-    for value in values:
-        if url_resolver is None:
-            tags.append(
-                str(
-                    format_html(
-                        '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                        'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>',
-                        value,
-                    ),
-                ),
-            )
-        else:
-            tags.append(
-                str(
-                    format_html(
-                        '<a href="{}" style="text-decoration: none;">'
-                        '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                        'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>'
-                        "</a>",
-                        url_resolver(value),
-                        value,
-                    ),
-                ),
-            )
-    return f'<div style="display: flex; flex-wrap: wrap;">{"".join(tags)}</div>'
-
-
-def render_property_links(prop_name: str, prop_info: dict[str, Any], machine_admin_url: str | None) -> str:
-    links = [
-        str(format_html('<a href="{}">Computed value</a>', get_live_config_url(prop_name))),
-    ]
-    if machine_admin_url:
-        links.append(str(format_html('<a href="{}">Edit override</a>', machine_admin_url)))
-
-    fallback = prop_info.get("x-fallback")
-    if isinstance(fallback, str) and fallback:
-        links.append(str(format_html('<a href="{}">Fallback: <code>{}</code></a>', get_live_config_url(fallback), fallback)))
-
-    aliases = prop_info.get("x-aliases") or []
-    if isinstance(aliases, list):
-        for alias in aliases:
-            if isinstance(alias, str) and alias:
-                links.append(str(format_html('<a href="{}">Alias: <code>{}</code></a>', get_live_config_url(alias), alias)))
-
-    default = prop_info.get("default")
-    if prop_name.endswith("_BINARY") and isinstance(default, str) and default:
-        links.append(str(format_html('<a href="{}">Binary: <code>{}</code></a>', get_environment_binary_url(default), default)))
-
-    return " &nbsp; ".join(links)
-
-
-def render_config_properties_html(properties: dict[str, Any], machine_admin_url: str | None) -> str:
-    header_links = [
-        str(format_html('<a href="{}">Dependencies</a>', ENVIRONMENT_BINARIES_BASE_URL)),
-        str(format_html('<a href="{}">Installed Binaries</a>', INSTALLED_BINARIES_BASE_URL)),
-    ]
-    if machine_admin_url:
-        header_links.insert(0, str(format_html('<a href="{}">Machine Config Editor</a>', machine_admin_url)))
-
-    cards = [
-        f'<div style="margin: 0 0 16px 0;">{" &nbsp; | &nbsp; ".join(header_links)}</div>',
-    ]
-
-    for prop_name, prop_info in properties.items():
-        prop_type = prop_info.get("type", "unknown")
-        if isinstance(prop_type, list):
-            prop_type = " | ".join(str(type_name) for type_name in prop_type)
-        prop_desc = prop_info.get("description", "")
-
-        default_html = ""
-        if "default" in prop_info:
-            default_html = str(
-                format_html(
-                    '<div style="margin-top: 6px;"><b>Default:</b> <code>{}</code></div>',
-                    prop_info["default"],
-                ),
-            )
-
-        description_html = prop_desc or mark_safe('<span style="color: #6e7781;">(no description)</span>')
-        cards.append(
-            str(
-                format_html(
-                    '<div style="margin: 0 0 14px 0; padding: 12px; background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 6px;">'
-                    '<div style="margin-bottom: 6px;">'
-                    '<a href="{}" style="font-weight: 600;"><code>{}</code></a>'
-                    ' <span style="color: #6e7781;">({})</span>'
-                    "</div>"
-                    '<div style="margin-bottom: 6px;">{}</div>'
-                    '<div style="font-size: 0.95em;">{}</div>'
-                    "{}"
-                    "</div>",
-                    get_live_config_url(prop_name),
-                    prop_name,
-                    prop_type,
-                    description_html,
-                    mark_safe(render_property_links(prop_name, prop_info, machine_admin_url)),
-                    mark_safe(default_html),
-                ),
-            ),
-        )
-
-    return "".join(cards)
-
-
-def render_hook_links_html(plugin_name: str, hooks: list[str], source: str) -> str:
-    if not hooks:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    items = []
-    for hook_name in hooks:
-        if source == "builtin":
-            items.append(
-                str(
-                    format_html(
-                        '<div style="margin: 0 0 8px 0;"><a href="{}" target="_blank" rel="noopener noreferrer"><code>{}</code></a></div>',
-                        get_plugin_hook_source_url(plugin_name, hook_name),
-                        hook_name,
-                    ),
-                ),
-            )
-        else:
-            items.append(
-                str(
-                    format_html(
-                        '<div style="margin: 0 0 8px 0;"><code>{}</code></div>',
-                        hook_name,
-                    ),
-                ),
-            )
-    return "".join(items)
-
-
 def render_binary_detail_description(name: str, merged: dict[str, Any], db_binary: Any) -> str:
     installed_binary_url = get_installed_binary_change_url(name, db_binary)
 
@@ -359,59 +111,32 @@ def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]:
 
 
 def get_db_binaries_by_name() -> dict[str, Binary]:
+    """Group Binary rows by a URL-safe canonical name.
+
+    Hooks occasionally emit ``BinaryEvent.name`` carrying an abspath rather
+    than a short binary name (see ``services/binary_service.py``). That used
+    to leak ``name='/Users/.../bin/foo'`` rows into the DB, which then broke
+    ``/admin/environment/binaries`` because the admin URL regex is
+    ``(?P<key>[^/]+)``. Canonicalize at the keying step so duplicates fold
+    into the real binary and the admin link key stays slash-free regardless
+    of legacy DB state.
+    """
+    from archivebox.machine.models import _canonical_binary_name
+
     grouped: dict[str, list[Binary]] = {}
     binary_name_aliases = {
         "youtube-dl": "yt-dlp",
     }
     for binary in Binary.objects.all():
-        canonical_name = binary_name_aliases.get(binary.name, binary.name)
+        canonical_name = _canonical_binary_name(binary.name)
+        canonical_name = binary_name_aliases.get(canonical_name, canonical_name)
+        if not canonical_name:
+            continue
         grouped.setdefault(canonical_name, []).append(binary)
 
     return {name: max(records, key=_binary_sort_key) for name, records in grouped.items()}
 
 
-def get_filesystem_plugins() -> dict[str, dict[str, Any]]:
-    """Discover plugins from filesystem directories."""
-    import json
-    from archivebox.hooks import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR
-
-    plugins = {}
-
-    for base_dir, source in [(BUILTIN_PLUGINS_DIR, "builtin"), (USER_PLUGINS_DIR, "user")]:
-        if not base_dir.exists():
-            continue
-
-        for plugin_dir in base_dir.iterdir():
-            if plugin_dir.is_dir() and not plugin_dir.name.startswith("_"):
-                plugin_id = f"{source}.{plugin_dir.name}"
-
-                # Find hook scripts
-                hooks = []
-                for ext in ("sh", "py", "js"):
-                    hooks.extend(plugin_dir.glob(f"on_*__*.{ext}"))
-
-                # Load config.json if it exists
-                config_file = plugin_dir / "config.json"
-                config_data = None
-                if config_file.exists():
-                    try:
-                        with open(config_file) as f:
-                            config_data = json.load(f)
-                    except (json.JSONDecodeError, OSError):
-                        config_data = None
-
-                plugins[plugin_id] = {
-                    "id": plugin_id,
-                    "name": plugin_dir.name,
-                    "path": str(plugin_dir),
-                    "source": source,
-                    "hooks": [str(h.name) for h in hooks],
-                    "config": config_data,
-                }
-
-    return plugins
-
-
 @render_with_table_view
 def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."
@@ -428,7 +153,7 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and getattr(binary, "is_valid", getattr(binary, "abspath", None)))
+        binary_is_valid = bool(binary and binary.is_valid)
 
         rows["Binary Name"].append(ItemLink(name, key=name))
 
@@ -455,9 +180,9 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
         "youtube-dl": "yt-dlp",
     }.get(key, key)
     db_binary = get_db_binaries_by_name().get(key)
-    binary_is_valid = bool(db_binary and getattr(db_binary, "is_valid", getattr(db_binary, "abspath", None)))
+    binary_is_valid = bool(db_binary and db_binary.is_valid)
     if binary_is_valid:
-        binary_data = db_binary.to_json() if hasattr(db_binary, "to_json") else db_binary.__dict__
+        binary_data = db_binary.to_json()
         section: SectionData = {
             "name": key,
             "description": mark_safe(render_binary_detail_description(key, binary_data, db_binary)),
@@ -496,137 +221,13 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     )
 
 
-@render_with_table_view
-def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    assert is_superuser(request), "Must be a superuser to view configuration settings."
-
-    rows = {
-        "Name": [],
-        "Source": [],
-        "Path": [],
-        "Hooks": [],
-        "Config": [],
-    }
-
-    plugins = get_filesystem_plugins()
-
-    for plugin_id, plugin in plugins.items():
-        rows["Name"].append(ItemLink(plugin["name"], key=plugin_id))
-        rows["Source"].append(plugin["source"])
-        rows["Path"].append(format_html("<code>{}</code>", plugin["path"]))
-        rows["Hooks"].append(", ".join(plugin["hooks"]) or "(none)")
-
-        # Show config status
-        if plugin.get("config"):
-            config_properties = plugin["config"].get("properties", {})
-            config_count = len(config_properties)
-            rows["Config"].append(f"✅ {config_count} properties" if config_count > 0 else "✅ present")
-        else:
-            rows["Config"].append("❌ none")
-
-    if not plugins:
-        # Show a helpful message when no plugins found
-        rows["Name"].append("(no plugins found)")
-        rows["Source"].append("-")
-        rows["Path"].append(mark_safe("<code>abx_plugins/plugins/</code> or <code>data/custom_plugins/</code>"))
-        rows["Hooks"].append("-")
-        rows["Config"].append("-")
-
-    return TableContext(
-        title="Installed plugins",
-        table=rows,
-    )
-
-
-@render_with_item_view
-def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-    assert is_superuser(request), "Must be a superuser to view configuration settings."
-
-    plugins = get_filesystem_plugins()
-
-    plugin = plugins.get(key)
-    if not plugin:
-        return ItemContext(
-            slug=key,
-            title=f"Plugin not found: {key}",
-            data=[],
-        )
-
-    # Base fields that all plugins have
-    docs_url = get_plugin_docs_url(plugin["name"])
-    machine_admin_url = get_machine_admin_url()
-    fields = {
-        "id": plugin["id"],
-        "name": plugin["name"],
-        "source": plugin["source"],
-    }
-
-    sections: list[SectionData] = [
-        {
-            "name": plugin["name"],
-            "description": format_html(
-                '<code>{}</code><br/><a href="{}" target="_blank" rel="noopener noreferrer">ABX Plugin Docs</a>',
-                plugin["path"],
-                docs_url,
-            ),
-            "fields": fields,
-            "help_texts": {},
-        },
-    ]
-
-    if plugin["hooks"]:
-        sections.append(
-            {
-                "name": "Hooks",
-                "description": mark_safe(render_hook_links_html(plugin["name"], plugin["hooks"], plugin["source"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-    if plugin.get("config"):
-        sections.append(
-            {
-                "name": "Plugin Metadata",
-                "description": mark_safe(render_plugin_metadata_html(plugin["config"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-        sections.append(
-            {
-                "name": "config.json",
-                "description": mark_safe(render_highlighted_json_block(plugin["config"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-        config_properties = plugin["config"].get("properties", {})
-        if config_properties:
-            sections.append(
-                {
-                    "name": "Config Properties",
-                    "description": mark_safe(render_config_properties_html(config_properties, machine_admin_url)),
-                    "fields": {},
-                    "help_texts": {},
-                },
-            )
-
-    return ItemContext(
-        slug=key,
-        title=plugin["name"],
-        data=sections,
-    )
-
-
 @render_with_table_view
 def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."
 
     rows = {
         "Name": [],
+        "Type": [],
         "State": [],
         "PID": [],
         "Started": [],
@@ -656,12 +257,115 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
             continue
         all_config[config_name] = config_data
 
-    # Add top row for supervisord process manager
+    # Collect every PID we plan to show so we can resolve them to Process rows
+    # in a single query. supervisord's per-worker description carries the pid
+    # in the form ``pid 12345, uptime 0:01:23`` (or just the bare ``pid``
+    # placeholder when stopped); we ignore non-numeric values.
+    process_items = supervisor.getAllProcessInfo()
+    if not isinstance(process_items, list):
+        process_items = []
+
+    def _parse_worker_pid_and_uptime(description: str) -> tuple[int | None, str]:
+        body = description.replace("pid ", "", 1)
+        pid_part, _, uptime_part = body.partition(", uptime ")
+        try:
+            return int(pid_part.strip()), uptime_part.strip()
+        except ValueError:
+            return None, ""
+
+    pids: set[int] = set()
+    supervisor_pid = supervisor.getPID()
+    if isinstance(supervisor_pid, int):
+        pids.add(supervisor_pid)
+    for proc_data in process_items:
+        if not isinstance(proc_data, dict):
+            continue
+        pid_int, _ = _parse_worker_pid_and_uptime(str(proc_data.get("description") or ""))
+        if pid_int is not None:
+            pids.add(pid_int)
+
+    pid_to_process_id: dict[int, str] = {}
+    pid_to_process_type: dict[int, str] = {}
+    if pids:
+        try:
+            from archivebox.machine.models import Machine, Process
+
+            for row in (
+                Process.objects.filter(machine=Machine.current(), pid__in=pids)
+                .order_by("pid", "-started_at", "-created_at")
+                .only("id", "pid", "process_type")
+            ):
+                if row.pid in pid_to_process_id:
+                    continue  # keep the most recent row per PID
+                pid_to_process_id[row.pid] = str(row.id)
+                pid_to_process_type[row.pid] = row.process_type
+
+        except Exception:
+            pass
+
+    def _pid_cell(pid_value: int | None, uptime_str: str = ""):
+        if pid_value is None:
+            return ""
+        pid_text = str(pid_value)
+        process_id = pid_to_process_id.get(pid_value)
+        if process_id:
+            link = format_html('<a href="/admin/machine/process/{}/change/">{}</a>', process_id, pid_text)
+        else:
+            link = format_html("{}", pid_text)
+        if uptime_str:
+            return format_html("{}, uptime {}", link, uptime_str)
+        return link
+
+    # Add top row for supervisord process manager. supervisord exposes its
+    # state + pid over XML-RPC but not its own start time / exit status / uptime,
+    # so we read those from the OS process (or fall back to the Process row
+    # recorded in _record_supervisord_process). Exit status stays blank while
+    # it's RUNNING — supervisord wouldn't be answering RPC if it had exited.
     rows["Name"].append(ItemLink("supervisord", key="supervisord"))
+    rows["Type"].append("supervisord")
     supervisor_state = supervisor.getState()
-    rows["State"].append(str(supervisor_state.get("statename") if isinstance(supervisor_state, dict) else ""))
-    rows["PID"].append(str(supervisor.getPID()))
-    rows["Started"].append("-")
+    state_name = str(supervisor_state.get("statename") if isinstance(supervisor_state, dict) else "")
+    rows["State"].append(state_name)
+
+    supervisor_started = ""
+    supervisor_uptime = ""
+    try:
+        import time as _time
+
+        import psutil
+
+        ps_proc = psutil.Process(supervisor_pid)
+        create_time = ps_proc.create_time()
+        supervisor_started = format_parsed_datetime(create_time)
+        seconds = max(int(_time.time() - create_time), 0)
+        hours, remainder = divmod(seconds, 3600)
+        minutes, secs = divmod(remainder, 60)
+        supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
+    except Exception:
+        try:
+            from archivebox.machine.models import Machine, Process
+
+            row = (
+                Process.objects.filter(
+                    machine=Machine.current(),
+                    process_type=Process.TypeChoices.SUPERVISORD,
+                    pid=supervisor_pid,
+                )
+                .order_by("-started_at")
+                .first()
+            )
+            if row and row.started_at:
+                supervisor_started = row.started_at.strftime("%Y-%m-%d %H:%M:%S")
+                seconds = max(int((timezone.now() - row.started_at).total_seconds()), 0)
+                hours, remainder = divmod(seconds, 3600)
+                minutes, secs = divmod(remainder, 60)
+                supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
+        except Exception:
+            pass
+
+    rows["PID"].append(_pid_cell(supervisor_pid if isinstance(supervisor_pid, int) else None, supervisor_uptime))
+    rows["Started"].append(supervisor_started or "-")
+
     rows["Command"].append("supervisord --configuration=tmp/supervisord.conf")
     rows["Logfile"].append(
         format_html(
@@ -670,12 +374,9 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
             "logs/supervisord.log",
         ),
     )
-    rows["Exit Status"].append("0")
+    rows["Exit Status"].append("" if state_name == "RUNNING" else "-")
 
     # Add a row for each worker process managed by supervisord
-    process_items = supervisor.getAllProcessInfo()
-    if not isinstance(process_items, list):
-        process_items = []
     for proc_data in process_items:
         if not isinstance(proc_data, dict):
             continue
@@ -684,10 +385,15 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
         proc_start = proc_data.get("start")
         proc_logfile = str(proc_data.get("stdout_logfile") or "")
         proc_config = all_config.get(proc_name, {})
+        pid_int, uptime_str = _parse_worker_pid_and_uptime(proc_description)
 
         rows["Name"].append(ItemLink(proc_name, key=proc_name))
+        # Prefer the Process row's process_type when we have one (e.g. "worker",
+        # "hook"); otherwise fall back to the generic "worker" label since
+        # everything in this loop is supervisord-managed.
+        rows["Type"].append(pid_to_process_type.get(pid_int, "worker") if pid_int else "worker")
         rows["State"].append(str(proc_data.get("statename") or ""))
-        rows["PID"].append(proc_description.replace("pid ", ""))
+        rows["PID"].append(_pid_cell(pid_int, uptime_str))
         rows["Started"].append(format_parsed_datetime(proc_start))
         rows["Command"].append(str(proc_config.get("command") or ""))
         rows["Logfile"].append(
@@ -810,7 +516,7 @@ def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
                 f.seek(0)
             last_lines = f.read().decode("utf-8", errors="replace").split("\n")
             non_empty_lines = [line for line in last_lines if line.strip()]
-            rows["Most Recent Lines"].append(non_empty_lines[-1])
+            rows["Most Recent Lines"].append(non_empty_lines[-1] if non_empty_lines else "")
 
     return TableContext(
         title="Debug Log files",

archivebox/core/admin_archiveresults.py

@@ -19,23 +19,21 @@ from django.urls import reverse, resolve
 from django.utils import timezone
 from django.utils.text import smart_split
 
-from archivebox.config import DATA_DIR
-from archivebox.config.common import get_config
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.base_models.admin import BaseModelAdmin
-from archivebox.hooks import get_plugin_icon
-from archivebox.core.host_utils import build_snapshot_url
+from archivebox.plugins.discovery import get_plugin_icon
+from archivebox.plugins.views import LIVE_PLUGIN_BASE_URL
+from archivebox.core.routes_util import build_snapshot_url
 from archivebox.core.widgets import InlineTagEditorWidget
-from archivebox.core.views import LIVE_PLUGIN_BASE_URL
-from archivebox.machine.env_utils import env_to_shell_exports
+from archivebox.machine.env_util import env_to_shell_exports
 
 
 from archivebox.core.models import ArchiveResult, Snapshot
 
 
 def _get_replay_source_url(result: ArchiveResult) -> str:
-    process = getattr(result, "process", None)
-    return str(getattr(process, "url", None) or result.snapshot.url or "")
+    process = result.process_record
+    return str((process.url if process else None) or result.snapshot.url or "")
 
 
 def build_abx_dl_display_command(result: ArchiveResult) -> str:
@@ -51,8 +49,8 @@ def build_abx_dl_display_command(result: ArchiveResult) -> str:
 
 def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str:
     display_command = build_abx_dl_display_command(result)
-    process = getattr(result, "process", None)
-    env_items = env_to_shell_exports(getattr(process, "env", None) or {})
+    process = result.process
+    env_items = env_to_shell_exports(process.env if process else {})
     if config is not None:
         result.snapshot._runtime_config = config
     snapshot_dir = shlex.quote(str(result.pwd or result.snapshot_dir))
@@ -62,7 +60,7 @@ def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str:
 
 
 def get_plugin_admin_url(plugin_name: str) -> str:
-    from archivebox.hooks import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, iter_plugin_dirs
+    from archivebox.plugins.discovery import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, iter_plugin_dirs
 
     plugin_dir = next((path.resolve() for path in iter_plugin_dirs() if path.name == plugin_name), None)
     if plugin_dir:
@@ -77,6 +75,12 @@ def get_plugin_admin_url(plugin_name: str) -> str:
     return f"{LIVE_PLUGIN_BASE_URL}builtin.{quote(plugin_name)}/"
 
 
+def get_process_link_label(process) -> str:
+    if process.pid:
+        return str(process.pid)
+    return str(process.id)[-8:]
+
+
 def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
     """Render a nice inline list view of archive results with status, plugin, output, and actions."""
 
@@ -105,6 +109,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         "failed": ("#991b1b", "#fee2e2"),  # red
         "queued": ("#6b7280", "#f3f4f6"),  # gray
         "started": ("#92400e", "#fef3c7"),  # amber
+        "paused": ("#1d4ed8", "#dbeafe"),  # blue
         "backoff": ("#92400e", "#fef3c7"),
         "skipped": ("#475569", "#f1f5f9"),
         "noresults": ("#475569", "#f1f5f9"),
@@ -134,27 +139,34 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         # Format timestamp
         end_time = result.end_ts.strftime("%Y-%m-%d %H:%M:%S") if result.end_ts else "-"
 
+        process = result.process_record
         process_display = "-"
-        if result.process_id and result.process:
+        if process:
+            process_url = html.escape(reverse("admin:machine_process_change", args=[process.id]), quote=True)
+            process_label = html.escape(get_process_link_label(process), quote=True)
             process_display = f'''
-                <a href="{reverse("admin:machine_process_change", args=[result.process_id])}"
+                <a href="{process_url}"
                    style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
-                   title="View process">{result.process.pid or "-"}</a>
+                   title="View process">{process_label}</a>
             '''
 
         machine_display = "-"
-        if result.process_id and result.process and result.process.machine_id:
+        if process and process.machine_id:
+            machine_url = html.escape(reverse("admin:machine_machine_change", args=[process.machine_id]), quote=True)
+            machine_label = html.escape(str(process.machine.hostname or ""), quote=True)
             machine_display = f'''
-                <a href="{reverse("admin:machine_machine_change", args=[result.process.machine_id])}"
+                <a href="{machine_url}"
                    style="color: #2563eb; text-decoration: none; font-size: 12px;"
-                   title="View machine">{result.process.machine.hostname}</a>
+                   title="View machine">{machine_label}</a>
             '''
 
         # Truncate output for display
-        full_output = result.output_str or "-"
-        output_display = full_output[:60]
-        if len(full_output) > 60:
-            output_display += "..."
+        full_output_raw = result.output_str_for_display() or "-"
+        output_display_raw = full_output_raw[:60]
+        if len(full_output_raw) > 60:
+            output_display_raw += "..."
+        full_output = html.escape(full_output_raw)
+        output_display = html.escape(output_display_raw)
 
         display_cmd = build_abx_dl_display_command(result)
         replay_cmd = build_abx_dl_replay_command(result, config=config)
@@ -162,15 +174,19 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         cmd_attr = html.escape(replay_cmd, quote=True)
 
         # Build output link - use embed_path() which checks output_files first
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else None
-        snapshot_id = str(getattr(result, "snapshot_id", ""))
+        embed_path = result.embed_path()
+        snapshot_id = str(result.snapshot_id)
         if embed_path and result.status == "succeeded":
             output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
         else:
             output_link = build_snapshot_url(snapshot_id, "", config=config)
+        output_link_attr = html.escape(output_link, quote=True)
 
         # Get version - try cmd_version field
-        version = result.cmd_version if result.cmd_version else "-"
+        version = html.escape(str(result.cmd_version if result.cmd_version else "-"), quote=True)
+        plugin_text = html.escape(str(result.plugin or ""), quote=True)
+        status_text = html.escape(str(status), quote=True)
+        pwd_text = html.escape(str(result.pwd or "-"), quote=True)
 
         # Unique ID for this row's expandable output
         row_id = f"output_{idx}_{str(result.id)[:8]}"
@@ -187,18 +203,18 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                 <td style="padding: 10px 12px; white-space: nowrap;">
                     <span style="display: inline-block; padding: 3px 10px; border-radius: 12px;
                                  font-size: 11px; font-weight: 600; text-transform: uppercase;
-                                 color: {color}; background: {bg};">{status}</span>
+                                 color: {color}; background: {bg};">{status_text}</span>
                 </td>
-                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{result.plugin}">
+                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{plugin_text}">
                     {icon}
                 </td>
                 <td style="padding: 10px 12px; font-weight: 500; color: #334155;">
-                        <a href="{output_link}" target="_blank"
+                        <a href="{output_link_attr}" target="_blank"
                            style="color: #334155; text-decoration: none;"
                        title="View output fullscreen"
                        onmouseover="this.style.color='#2563eb'; this.style.textDecoration='underline';"
                        onmouseout="this.style.color='#334155'; this.style.textDecoration='none';">
-                        {result.plugin}
+                        {plugin_text}
                     </a>
                 </td>
                 <td style="padding: 10px 12px; max-width: 280px;">
@@ -225,7 +241,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                 </td>
                 <td style="padding: 10px 8px; white-space: nowrap;">
                     <div style="display: flex; gap: 4px;">
-                        <a href="{output_link}" target="_blank"
+                        <a href="{output_link_attr}" target="_blank"
                            style="padding: 4px 8px; background: #f1f5f9; border-radius: 4px; color: #475569; text-decoration: none; font-size: 11px;"
                            title="View output">📄</a>
                         <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
@@ -244,7 +260,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                             <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
                                 <span style="margin-right: 16px;"><b>ID:</b> <code>{str(result.id)}</code></span>
                                 <span style="margin-right: 16px;"><b>Version:</b> <code>{version}</code></span>
-                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{result.pwd or "-"}</code></span>
+                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{pwd_text}</code></span>
                             </div>
                             <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
                                 <b>Output:</b>
@@ -352,8 +368,7 @@ class ArchiveResultInline(admin.TabularInline):
     def get_formset(self, request, obj=None, **kwargs):
         formset = super().get_formset(request, obj, **kwargs)
         snapshot = self.get_parent_object_from_request(request)
-        form_class = getattr(formset, "form", None)
-        base_fields = getattr(form_class, "base_fields", {})
+        base_fields = formset.form.base_fields
         snapshot_output_dir = str(snapshot.output_dir) if snapshot else ""
 
         # import ipdb; ipdb.set_trace()
@@ -489,14 +504,12 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     def change_view(self, request, object_id, form_url="", extra_context=None):
         self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         return super().change_view(request, object_id, form_url, extra_context)
 
     def changelist_view(self, request, extra_context=None):
         self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         saved_list_per_page = self.list_per_page
-        self.list_per_page = min(max(5, request.archivebox_config.SNAPSHOTS_PER_PAGE), 5000)
+        self.list_per_page = request.archivebox_config.SNAPSHOTS_PER_PAGE
         try:
             return super().changelist_view(request, extra_context)
         finally:
@@ -520,7 +533,6 @@ class ArchiveResultAdmin(BaseModelAdmin):
             super()
             .get_queryset(request)
             .defer(
-                "config",
                 "notes",
                 "output_json",
             )
@@ -571,20 +583,20 @@ class ArchiveResultAdmin(BaseModelAdmin):
         return queryset.filter(reduce(and_, filters)).distinct(), True
 
     def get_snapshot_view_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        return build_snapshot_url(str(result.snapshot_id), request=request, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return build_snapshot_url(str(result.snapshot_id), request=request, config=request.archivebox_config)
 
     def get_output_view_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
-        output_path = result.embed_path() if hasattr(result, "embed_path") else None
+        request = self.request
+        config = request.archivebox_config
+        output_path = result.embed_path()
         if not output_path:
             output_path = result.plugin or ""
         return build_snapshot_url(str(result.snapshot_id), output_path, request=request, config=config)
 
     def get_output_files_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
+        request = self.request
+        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=request.archivebox_config)}/?files=1"
 
     def get_output_zip_url(self, result: ArchiveResult) -> str:
         return f"{self.get_output_files_url(result)}&download=zip"
@@ -610,10 +622,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
     )
     def snapshot_info(self, result):
         snapshot_id = str(result.snapshot_id)
-        request = getattr(self, "request", None)
+        request = self.request
         return format_html(
             '<a href="{}"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
             snapshot_id[:8],
             result.snapshot.bookmarked_at.strftime("%Y-%m-%d %H:%M"),
             result.snapshot.url[:128],
@@ -660,32 +672,33 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     @admin.display(description="Process", ordering="process__pid")
     def process_link(self, result):
-        if not result.process_id:
+        process = result.process_record
+        if not process:
             return "-"
-        process_label = result.process.pid if result.process and result.process.pid else "-"
+        process_label = get_process_link_label(process)
         return format_html(
             '<a href="{}"><code>{}</code></a>',
-            reverse("admin:machine_process_change", args=[result.process_id]),
+            reverse("admin:machine_process_change", args=[process.id]),
             process_label,
         )
 
     @admin.display(description="Machine", ordering="process__machine__hostname")
     def machine_link(self, result):
-        if not result.process_id or not result.process or not result.process.machine_id:
+        process = result.process_record
+        if not process or not process.machine_id:
             return "-"
-        machine = result.process.machine
+        machine = process.machine
         return format_html(
-            '<a href="{}"><code>{}</code> {}</a>',
+            '<a href="{}">{}</a>',
             reverse("admin:machine_machine_change", args=[machine.id]),
-            str(machine.id)[:8],
             machine.hostname,
         )
 
     @admin.display(description="Command")
     def cmd_str(self, result):
-        request = getattr(self, "request", None)
+        request = self.request
         display_cmd = build_abx_dl_display_command(result)
-        replay_cmd = build_abx_dl_replay_command(result, config=getattr(request, "archivebox_config", None))
+        replay_cmd = build_abx_dl_replay_command(result, config=request.archivebox_config)
         return format_html(
             """
             <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
@@ -706,30 +719,30 @@ class ArchiveResultAdmin(BaseModelAdmin):
         )
 
     def output_display(self, result):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         # Determine output link path - use embed_path() which checks output_files
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else None
+        embed_path = result.embed_path()
         output_path = embed_path if (result.status == "succeeded" and embed_path) else "index.html"
         snapshot_id = str(result.snapshot_id)
         return format_html(
             '<a href="{}" class="output-link">↗️</a><pre>{}</pre>',
             build_snapshot_url(snapshot_id, output_path, request=request, config=config),
-            result.output_str,
+            result.output_str_for_display(),
         )
 
     @admin.display(description="Output", ordering="output_str")
     def output_str_display(self, result):
-        output_text = str(result.output_str or "").strip()
+        output_text = str(result.output_str_for_display() or "").strip()
         if not output_text:
             return "-"
 
-        request = getattr(self, "request", None)
-        live_path = result.embed_path() if hasattr(result, "embed_path") else None
+        request = self.request
+        live_path = result.embed_path()
         if live_path:
             return format_html(
                 '<a href="{}" title="{}"><code>{}</code></a>',
-                build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=getattr(request, "archivebox_config", None)),
+                build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=request.archivebox_config),
                 output_text,
                 output_text,
             )
@@ -781,18 +794,18 @@ class ArchiveResultAdmin(BaseModelAdmin):
         )
 
     def output_summary(self, result):
-        snapshot_dir = Path(DATA_DIR) / str(result.pwd).split("data/", 1)[-1]
+        snapshot_dir = Path(result.snapshot.output_dir)
         output_html = format_html(
             '<pre style="display: inline-block">{}</pre><br/>',
-            result.output_str,
+            result.output_str_for_display(),
         )
         snapshot_id = str(result.snapshot_id)
-        request = getattr(self, "request", None)
+        request = self.request
         output_html += format_html(
             '<a href="{}#all">See result files ...</a><br/><pre><code>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
         )
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else ""
+        embed_path = result.embed_path() or ""
         path_from_embed = snapshot_dir / (embed_path or "")
         output_html += format_html(
             '<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>',

archivebox/core/admin_site.py

@@ -3,7 +3,15 @@ __package__ = "archivebox.core"
 from typing import TYPE_CHECKING, Any
 
 from django.contrib import admin
+from django.contrib.auth import REDIRECT_FIELD_NAME
+from django.contrib.auth.decorators import login_not_required
+from django.contrib.auth.views import LoginView
 from django.db import DatabaseError, connection
+from django.http import HttpResponseRedirect
+from django.urls import reverse
+from django.utils.decorators import method_decorator
+from django.utils.translation import gettext as _
+from django.views.decorators.cache import never_cache
 from admin_data_views.admin import (
     admin_data_index_view as adv_admin_data_index_view,
     get_admin_data_urls as adv_get_admin_data_urls,
@@ -12,6 +20,7 @@ from admin_data_views.admin import (
 
 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH
+from archivebox.core.routes_util import is_allowed_archivebox_redirect_url
 
 if TYPE_CHECKING:
     from django.http import HttpRequest
@@ -21,6 +30,17 @@ if TYPE_CHECKING:
     from admin_data_views.typing import AppDict
 
 
+class ArchiveBoxLoginView(LoginView):
+    def get_redirect_url(self) -> str:
+        redirect_to = self.request.POST.get(
+            self.redirect_field_name,
+            self.request.GET.get(self.redirect_field_name),
+        )
+        if is_allowed_archivebox_redirect_url(redirect_to, request=self.request):
+            return redirect_to
+        return ""
+
+
 class ArchiveBoxAdmin(admin.AdminSite):
     site_header = "ArchiveBox"
     index_title = "Admin Views"
@@ -72,6 +92,34 @@ class ArchiveBoxAdmin(admin.AdminSite):
     def admin_data_index_view(self, request: "HttpRequest", **kwargs: Any) -> "TemplateResponse":
         return adv_admin_data_index_view(self, request, **kwargs)
 
+    @method_decorator(never_cache)
+    @login_not_required
+    def login(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
+        if request.method == "GET" and self.has_permission(request):
+            return HttpResponseRedirect(reverse("admin:index", current_app=self.name))
+
+        from django.contrib.admin.forms import AdminAuthenticationForm
+
+        context = {
+            **self.each_context(request),
+            "title": _("Log in"),
+            "subtitle": None,
+            "app_path": request.get_full_path(),
+            "username": request.user.get_username(),
+        }
+        if REDIRECT_FIELD_NAME not in request.GET and REDIRECT_FIELD_NAME not in request.POST:
+            context[REDIRECT_FIELD_NAME] = reverse("admin:index", current_app=self.name)
+        context.update(extra_context or {})
+
+        index_path = reverse("admin:index", current_app=self.name)
+        request.current_app = self.name
+        return ArchiveBoxLoginView.as_view(
+            extra_context=context,
+            authentication_form=self.login_form or AdminAuthenticationForm,
+            template_name=self.login_template or "admin/login.html",
+            next_page=index_path,
+        )(request)
+
     def index(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
         response = super().index(request, extra_context)
         if connection.vendor != "sqlite":

archivebox/core/admin_tags.py

@@ -13,7 +13,7 @@ from django.utils.safestring import mark_safe
 
 from archivebox.base_models.admin import BaseModelAdmin
 from archivebox.core.models import SnapshotTag, Tag
-from archivebox.core.tag_utils import (
+from archivebox.core.tag_util import (
     TAG_HAS_SNAPSHOTS_CHOICES,
     TAG_SORT_CHOICES,
     build_tag_cards,
@@ -24,7 +24,7 @@ from archivebox.core.tag_utils import (
     normalize_has_snapshots_filter,
     normalize_tag_sort,
 )
-from archivebox.core.host_utils import build_snapshot_url
+from archivebox.core.routes_util import build_snapshot_url
 
 
 class TagInline(admin.TabularInline):
@@ -117,7 +117,7 @@ class TagAdmin(BaseModelAdmin):
         sort = normalize_tag_sort((request.GET.get("sort") or "created_desc").strip())
         created_by = normalize_created_by_filter((request.GET.get("created_by") or "").strip())
         year = normalize_created_year_filter((request.GET.get("year") or "").strip())
-        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "yes").strip())
+        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "all").strip())
         context = {
             **self.admin_site.each_context(request),
             **(extra_context or {}),
@@ -231,7 +231,7 @@ class TagAdmin(BaseModelAdmin):
 
     @admin.display(description="Snapshots", ordering="num_snapshots")
     def num_snapshots(self, tag: Tag):
-        count = getattr(tag, "num_snapshots", None)
+        count = tag.__dict__.get("num_snapshots")
         if count is None:
             count = tag.snapshot_set.count()
         return format_html(

archivebox/core/admin_users.py

@@ -47,7 +47,7 @@ class CustomUserAdmin(UserAdmin):
 
     def snapshot_count_badge(self, obj):
         snapshots_url = f"/admin/core/snapshot/?created_by__id__exact={obj.pk}"
-        snapshot_count = getattr(obj, "snapshot_count", 0)
+        snapshot_count = obj.__dict__.get("snapshot_count", 0)
         snapshot_label = "snapshot" if snapshot_count == 1 else "snapshots"
         return format_html(
             (

archivebox/core/apps.py

@@ -16,24 +16,15 @@ class CoreConfig(AppConfig):
         from archivebox.core.admin_site import register_admin_site
 
         register_admin_site()
+        from archivebox.base_models.models import ModelWithOutputDir
+
+        ModelWithOutputDir.register_delete_signal()
 
         # Import models to register state machines with the registry
         # Skip during makemigrations to avoid premature state machine access
         if "makemigrations" not in sys.argv:
             from archivebox.core import models  # noqa: F401
 
-        pidfile = os.environ.get("ARCHIVEBOX_RUNSERVER_PIDFILE")
-        if pidfile:
-            should_write_pid = True
-            if os.environ.get("ARCHIVEBOX_AUTORELOAD") == "1":
-                should_write_pid = os.environ.get(DJANGO_AUTORELOAD_ENV) == "true"
-            if should_write_pid:
-                try:
-                    with open(pidfile, "w") as handle:
-                        handle.write(str(os.getpid()))
-                except Exception:
-                    pass
-
         def _should_prepare_runtime() -> bool:
             if os.environ.get("ARCHIVEBOX_RUNSERVER") == "1":
                 if os.environ.get("ARCHIVEBOX_AUTORELOAD") == "1":
@@ -42,8 +33,13 @@ class CoreConfig(AppConfig):
             return False
 
         if _should_prepare_runtime():
-            from archivebox.machine.models import Process, Machine
+            from archivebox.config import CONSTANTS
+            from archivebox.machine.models import Process
 
-            Process.cleanup_stale_running()
-            Process.cleanup_orphaned_workers()
-            Machine.current()
+            Process.current().mark_running(
+                process_type=Process.TypeChoices.WORKER,
+                worker_type="worker_runserver",
+                pwd=str(CONSTANTS.DATA_DIR),
+                url=os.environ.get("ARCHIVEBOX_RUNSERVER_BIND_URL") or "",
+                timeout=CONSTANTS.MAX_HOOK_RUNTIME_SECONDS,
+            )

archivebox/core/asgi.py

@@ -12,6 +12,65 @@ from django.core.asgi import get_asgi_application
 
 setup_django(in_memory_db=False, check_db=True)
 
+
+def _patch_thread_sensitive_context_shutdown() -> None:
+    """Stop ``ThreadSensitiveContext.__aexit__`` from blocking the daphne loop.
+
+    Django 6.0's ASGIHandler wraps every request in ``async with
+    ThreadSensitiveContext():`` (django/core/handlers/asgi.py:169). On exit
+    asgiref calls ``executor.shutdown()`` with the default ``wait=True``
+    (asgiref/sync.py:148), which is a *synchronous* ``Thread.join()`` inside
+    an async function — so it blocks the daphne event loop until the
+    executor's worker thread exits.
+
+    That's normally fine because the request handler has already awaited
+    every ``sync_to_async`` it submitted, so the worker is idle and dies as
+    soon as the shutdown sentinel reaches it. The blocking turns into a
+    problem when a client disconnects mid-request:
+
+    * ``SyncToAsync.__call__`` shields the executor work with
+      ``await asyncio.shield(exec_coro)`` (asgiref/sync.py:506) so that the
+      sync DB call doesn't get torn down halfway through.
+    * On cancellation it calls ``exec_coro.cancel()`` (line 522) which only
+      flips the asyncio ``Future`` to cancelled — the underlying thread
+      keeps running the SQL query.
+    * Control unwinds to ``__aexit__`` while the orphaned thread is still
+      mid-query. ``shutdown(wait=True)`` then blocks the event loop until
+      that orphan finishes.
+
+    Under heavy SQLite contention (the load-test scenario that surfaced
+    this on cabbage) those orphan threads can take 30 seconds each waiting
+    for write locks, and the daphne loop is single-threaded — so every
+    such orphan stalls every other in-flight request, healthchecks time
+    out, and the container goes ``unhealthy``.
+
+    Switching to ``shutdown(wait=False)`` queues the sentinel and returns
+    immediately; the worker thread still exits cleanly once its current
+    task finishes, and asgiref's ``WeakKeyDictionary`` releases the
+    executor as soon as the request's context is GC'd. No per-request
+    teardown guarantee is lost — there was no caller relying on it.
+    """
+    from asgiref import sync as _asgiref_sync
+
+    original_aexit = _asgiref_sync.ThreadSensitiveContext.__aexit__
+
+    async def __aexit__(self, exc, value, tb):  # type: ignore[no-redef]
+        if not self.token:
+            return
+        executor = _asgiref_sync.SyncToAsync.context_to_thread_executor.pop(self, None)
+        if executor is not None:
+            executor.shutdown(wait=False)
+        _asgiref_sync.SyncToAsync.thread_sensitive_context.reset(self.token)
+
+    # Idempotent: only patch once even if asgi.py is reloaded.
+    if getattr(original_aexit, "_archivebox_patched", False):
+        return
+    __aexit__._archivebox_patched = True  # type: ignore[attr-defined]
+    _asgiref_sync.ThreadSensitiveContext.__aexit__ = __aexit__
+
+
+_patch_thread_sensitive_context_shutdown()
+
 # Standard Django ASGI application (no websockets/channels needed)
 application = get_asgi_application()
 

archivebox/core/context_processors.py

@@ -0,0 +1,9 @@
+from archivebox.config import VERSION
+from archivebox.config.version import get_COMMIT_HASH
+
+
+def archivebox_globals(request):
+    return {
+        "VERSION": VERSION,
+        "STATIC_CACHE_KEY": (get_COMMIT_HASH() or VERSION or "dev").strip(),
+    }

archivebox/core/forms.py

@@ -1,20 +1,23 @@
 __package__ = "archivebox.core"
 
-import json
-from collections.abc import Iterable, Mapping
-from pathlib import Path
-from typing import Any
+import re
+from decimal import Decimal, InvalidOperation, ROUND_CEILING
 
 from django import forms
-from django.utils.html import format_html
 
 from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
-from taggit.utils import edit_string_for_tags, parse_tags
 from archivebox.base_models.admin import KeyValueWidget
-from archivebox.crawls.schedule_utils import validate_schedule
-from archivebox.config.common import get_config
+from archivebox.crawls.schedule_util import validate_schedule
+from archivebox.config.common import get_config, parse_delete_after
+from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
-from archivebox.hooks import get_plugins, discover_plugin_configs, get_plugin_icon
+from archivebox.plugins.discovery import get_plugins
+from archivebox.plugins.forms import (
+    PLUGIN_GROUP_DEFINITIONS,
+    TIMEOUT_INPUT_PATTERN,
+    PluginConfigFormMixin,
+    get_choice_field,
+)
 from archivebox.personas.models import Persona
 
 DEPTH_CHOICES = (
@@ -25,490 +28,86 @@ DEPTH_CHOICES = (
     ("4", "depth = 4 (+ URLs four hops away)"),
 )
 
-PLUGIN_CONFIG_FIELD_PREFIX = "plugin_config__"
-PLUGIN_GROUP_DEFINITIONS = (
-    (
-        "main_plugins",
-        "Main",
-        "",
-        "",
-        "",
-        (
-            "dom",
-            "screenshot",
-            "pdf",
-            "singlefile",
-            "wget",
-            "archivedotorg",
-            "chrome_mhtml",
-        ),
-    ),
-    (
-        "page_setup_plugins",
-        "Page Setup",
-        "",
-        "",
-        "",
-        (
-            "chrome",
-            "infiniscroll",
-            "modalcloser",
-            "ublock",
-            "istilldontcareaboutcookies",
-            "twocaptcha",
-            "claudechrome",
-        ),
-    ),
-    (
-        "media_plugins",
-        "Media",
-        "",
-        "",
-        "",
-        (
-            "staticfile",
-            "responses",
-            "ytdlp",
-            "gallerydl",
-            "git",
-        ),
-    ),
-    (
-        "text_plugins",
-        "Text",
-        "",
-        "",
-        "",
-        (
-            "readability",
-            "htmltotext",
-            "defuddle",
-            "forumdl",
-            "mercury",
-            "trafilatura",
-            "liteparse",
-            "opendataloader",
-            "papersdl",
-        ),
-    ),
-    (
-        "metadata_plugins",
-        "Metadata",
-        "",
-        "",
-        "",
-        (
-            "title",
-            "favicon",
-            "headers",
-            "redirects",
-            "accessibility",
-            "consolelog",
-            "sslcerts",
-            "dns",
-            "seo",
-            "hashes",
-        ),
-    ),
-    (
-        "postprocessing_plugins",
-        "Postprocessing",
-        "",
-        "",
-        "",
-        (
-            "parse_dom_outlinks",
-            "parse_html_urls",
-            "parse_jsonl_urls",
-            "parse_netscape_urls",
-            "parse_rss_urls",
-            "parse_txt_urls",
-            "claudecode",
-            "claudecodecleanup",
-            "claudecodeextract",
-        ),
-    ),
-)
-HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
-    "apt",
-    "base",
-    "bash",
-    "brew",
-    "cargo",
-    "chromewebstore",
-    "env",
-    "media",
-    "npm",
-    "pip",
-    "puppeteer",
-    "search_backend_ripgrep",
-    "search_backend_sonic",
-    "search_backend_sqlite",
-    "ssl",
-}
+
+def _split_strip(value: str, delimiter: str) -> list[str]:
+    return [part.strip() for part in value.split(delimiter) if part.strip()]
 
 
-def get_plugin_choices():
-    """Get available extractor plugins from discovered hooks."""
-    return [(name, name) for name in get_plugins()]
+def parse_tag_string(value: str | None) -> list[str]:
+    """Parse the legacy tag-editing format used by the admin tag field."""
+    if not value:
+        return []
+
+    if "," not in value and '"' not in value:
+        return sorted(set(_split_strip(value, " ")))
+
+    tags: list[str] = []
+    buffer: list[str] = []
+    deferred_chunks: list[str] = []
+    saw_unquoted_comma = False
+    in_quote = False
+    chars = iter(value)
+
+    try:
+        while True:
+            char = next(chars)
+            if char == '"':
+                if buffer:
+                    deferred_chunks.append("".join(buffer))
+                    buffer = []
+                in_quote = True
+                char = next(chars)
+                while char != '"':
+                    buffer.append(char)
+                    char = next(chars)
+                tag = "".join(buffer).strip()
+                if tag:
+                    tags.append(tag)
+                buffer = []
+                in_quote = False
+            else:
+                if not saw_unquoted_comma and char == ",":
+                    saw_unquoted_comma = True
+                buffer.append(char)
+    except StopIteration:
+        if buffer:
+            if in_quote and "," in buffer:
+                saw_unquoted_comma = True
+            deferred_chunks.append("".join(buffer))
+
+    delimiter = "," if saw_unquoted_comma else " "
+    for chunk in deferred_chunks:
+        tags.extend(_split_strip(chunk, delimiter))
+
+    return sorted(set(tags))
 
 
-def get_plugin_choice_label(plugin_name: str, plugin_configs: dict[str, dict]) -> str:
-    schema = plugin_configs.get(plugin_name, {})
-    description = str(schema.get("description") or "").strip()
-    if not description:
-        return plugin_name
-    icon_html = get_plugin_icon(plugin_name)
-
-    return format_html(
-        '<span class="plugin-choice-icon">{}</span><span class="plugin-choice-name">{}</span><a class="plugin-choice-description" href="https://archivebox.github.io/abx-plugins/#{}" target="_blank" rel="noopener noreferrer">{}</a>',
-        icon_html,
-        plugin_name,
-        plugin_name,
-        description,
-    )
-
-
-def get_choice_field(form: forms.Form, name: str) -> forms.ChoiceField:
-    field = form.fields[name]
-    if not isinstance(field, forms.ChoiceField):
-        raise TypeError(f"{name} must be a ChoiceField")
-    return field
-
-
-def _plugin_config_input_name(plugin_name: str, config_key: str) -> str:
-    return f"{PLUGIN_CONFIG_FIELD_PREFIX}{plugin_name}__{config_key}"
-
-
-def _schema_types(schema: Mapping[str, Any]) -> list[str]:
-    raw_type = schema.get("type") or "string"
-    if isinstance(raw_type, list):
-        return [str(item) for item in raw_type]
-    return [str(raw_type)]
-
-
-def _jsonish(value: Any) -> str:
-    if isinstance(value, str):
-        return value
-    return json.dumps(value, sort_keys=True, default=str)
-
-
-def _same_config_value(left: Any, right: Any) -> bool:
-    return json.dumps(left, sort_keys=True, default=str) == json.dumps(right, sort_keys=True, default=str)
-
-
-def _coerce_plugin_config_value(raw_value: Any, schema: Mapping[str, Any]) -> Any:
-    schema_types = _schema_types(schema)
-
-    if "boolean" in schema_types:
-        if isinstance(raw_value, bool):
-            return raw_value
-        value = str(raw_value).strip().lower()
-        if value in {"true", "1", "yes", "on"}:
-            return True
-        if value in {"false", "0", "no", "off", ""}:
-            return False
-        raise forms.ValidationError("Must be true or false.")
-
-    if "integer" in schema_types:
-        value = int(str(raw_value).strip())
-        minimum = schema.get("minimum")
-        maximum = schema.get("maximum")
-        if minimum is not None and value < int(minimum):
-            raise forms.ValidationError(f"Must be at least {minimum}.")
-        if maximum is not None and value > int(maximum):
-            raise forms.ValidationError(f"Must be at most {maximum}.")
-        return value
-
-    if "number" in schema_types:
-        value = float(str(raw_value).strip())
-        minimum = schema.get("minimum")
-        maximum = schema.get("maximum")
-        if minimum is not None and value < float(minimum):
-            raise forms.ValidationError(f"Must be at least {minimum}.")
-        if maximum is not None and value > float(maximum):
-            raise forms.ValidationError(f"Must be at most {maximum}.")
-        return value
-
-    if "array" in schema_types:
-        if isinstance(raw_value, list):
-            return raw_value
-        value = str(raw_value).strip()
-        if not value:
-            return []
-        if value.startswith("["):
-            parsed = json.loads(value)
-            if not isinstance(parsed, list):
-                raise forms.ValidationError("Must be a JSON array.")
-            return parsed
-        return [item.strip() for item in value.replace(",", "\n").splitlines() if item.strip()]
-
-    if "object" in schema_types:
-        value = str(raw_value).strip()
-        if not value:
-            return {}
-        parsed = json.loads(value)
-        if not isinstance(parsed, dict):
-            raise forms.ValidationError("Must be a JSON object.")
-        return parsed
-
-    value = str(raw_value)
-    enum = schema.get("enum")
-    if isinstance(enum, list) and enum and value not in {str(item) for item in enum}:
-        raise forms.ValidationError(f"Must be one of: {', '.join(str(item) for item in enum)}.")
-    return value
-
-
-class PluginConfigFormMixin:
-    plugin_groups: list[dict[str, Any]]
-
-    def build_plugin_groups(self, runtime_config: Mapping[str, Any] | None = None) -> None:
-        all_plugins = get_plugins()
-        plugin_configs = discover_plugin_configs()
-        runtime_config = runtime_config or get_config()
-        self.plugin_config_binary_urls = get_plugin_config_binary_urls(runtime_config)
-        grouped_plugins = set().union(*(group[-1] for group in PLUGIN_GROUP_DEFINITIONS))
-        other_plugins = tuple(sorted(set(all_plugins) - grouped_plugins - HIDDEN_PLUGIN_CONFIG_UI_PLUGINS))
-
-        for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
-            if field_name in self.fields:
-                get_choice_field(self, field_name).choices = [
-                    (p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins
-                ]
-
-        if "other_plugins" in self.fields:
-            get_choice_field(self, "other_plugins").choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in other_plugins]
-
-        group_specs = (
-            *PLUGIN_GROUP_DEFINITIONS,
-            ("other_plugins", "Other", "", "", "", other_plugins),
-        )
-        self.plugin_groups = [
-            {
-                "field_name": field_name,
-                "title": title,
-                "note": note,
-                "dom_id": dom_id,
-                "select_all_group": select_all_group,
-                "show_selectors": field_name in self.fields,
-                "plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config),
-            }
-            for field_name, title, note, dom_id, select_all_group, plugin_names in group_specs
-            if any(plugin in all_plugins for plugin in plugin_names)
-        ]
-
-    def _build_plugin_cards(
-        self,
-        field_name: str,
-        plugin_names: Iterable[str],
-        plugin_configs: dict[str, dict[str, Any]],
-        runtime_config: Mapping[str, Any],
-    ) -> list[dict[str, Any]]:
-        if field_name in self.fields:
-            choices = list(get_choice_field(self, field_name).choices)
-            selected_values = set(self.data.getlist(field_name)) if self.is_bound else set(get_choice_field(self, field_name).initial or [])
-        else:
-            all_plugins = get_plugins()
-            choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins]
-            selected_values = set()
-
-        cards = []
-        for index, (plugin_name, label) in enumerate(choices):
-            schema = plugin_configs.get(str(plugin_name), {})
-            properties = schema.get("properties") or {}
-            enabled_config_key = f"{str(plugin_name).upper()}_ENABLED"
-            enabled_prop_schema = properties.get(enabled_config_key)
-            if not isinstance(enabled_prop_schema, dict) or "boolean" not in _schema_types(enabled_prop_schema):
-                enabled_config_key = ""
-            config_fields = [
-                self._build_plugin_config_field(str(plugin_name), str(config_key), prop_schema, runtime_config)
-                for config_key, prop_schema in properties.items()
-                if isinstance(prop_schema, dict)
-            ]
-            cards.append(
-                {
-                    "name": str(plugin_name),
-                    "label": label,
-                    "checked": str(plugin_name) in selected_values,
-                    "checkbox_id": f"id_{field_name}_{index}",
-                    "enabled_config_key": enabled_config_key,
-                    "description": str(schema.get("description") or "").strip(),
-                    "required_plugins": [str(item) for item in schema.get("required_plugins") or []],
-                    "required_binaries_count": len(schema.get("required_binaries") or []),
-                    "config_fields": config_fields,
-                    "config_count": len(config_fields),
-                },
-            )
-        return cards
-
-    def _build_plugin_config_field(
-        self,
-        plugin_name: str,
-        config_key: str,
-        prop_schema: Mapping[str, Any],
-        runtime_config: Mapping[str, Any],
-    ) -> dict[str, Any]:
-        schema_types = _schema_types(prop_schema)
-        enum = prop_schema.get("enum")
-        input_name = _plugin_config_input_name(plugin_name, config_key)
-        current_value = runtime_config.get(config_key, prop_schema.get("default", ""))
-        if self.is_bound and input_name in self.data:
-            try:
-                current_value = _coerce_plugin_config_value(self.data.get(input_name), prop_schema)
-            except (TypeError, ValueError, json.JSONDecodeError, forms.ValidationError):
-                current_value = self.data.get(input_name)
-
-        default_value = prop_schema.get("default", "")
-        fallback_key = prop_schema.get("x-fallback")
-        default_display = f"{{{fallback_key}}}" if fallback_key else default_value
-        is_sensitive = bool(prop_schema.get("x-sensitive"))
-        input_value = "" if is_sensitive else _jsonish(current_value)
-        field_kind = "text"
-        input_type = "text"
-        options = []
-
-        if "boolean" in schema_types:
-            field_kind = "boolean"
-            input_value = "true" if bool(current_value) else "false"
-        elif isinstance(enum, list) and enum:
-            field_kind = "select"
-            options = [
-                {
-                    "value": str(option),
-                    "label": str(option),
-                    "selected": str(option) == str(current_value),
-                }
-                for option in enum
-            ]
-        elif "integer" in schema_types or "number" in schema_types:
-            field_kind = "number"
-            input_type = "number"
-        elif "array" in schema_types or "object" in schema_types:
-            field_kind = "json"
-            input_value = "" if is_sensitive else json.dumps(current_value, indent=2, sort_keys=True, default=str)
-        elif is_sensitive:
-            input_type = "password"
-
-        return {
-            "key": config_key,
-            "input_name": input_name,
-            "kind": field_kind,
-            "input_type": input_type,
-            "value": input_value,
-            "checked": bool(current_value),
-            "options": options,
-            "description": str(prop_schema.get("description") or "").strip(),
-            "default": _jsonish(default_display),
-            "current": "configured" if is_sensitive and current_value else _jsonish(current_value),
-            "current_url": self.plugin_config_binary_urls.get(config_key, "") if str(config_key).endswith("_BINARY") else "",
-            "is_sensitive": is_sensitive,
-            "minimum": prop_schema.get("minimum"),
-            "maximum": prop_schema.get("maximum"),
-            "pattern": prop_schema.get("pattern"),
-            "type_label": " / ".join(schema_types),
-        }
-
-    def clean_plugin_config_overrides(self, effective_config: Mapping[str, Any] | None = None) -> dict[str, Any]:
-        if not self.is_bound:
-            return {}
-
-        effective_config = effective_config or get_config()
-        overrides: dict[str, Any] = {}
-        sources: dict[str, str] = {}
-
-        for plugin_name, schema in discover_plugin_configs().items():
-            for config_key, prop_schema in (schema.get("properties") or {}).items():
-                if not isinstance(prop_schema, dict):
-                    continue
-
-                input_name = _plugin_config_input_name(plugin_name, config_key)
-                if input_name not in self.data:
-                    continue
-
-                raw_value: Any = self.data.get(input_name)
-                if "array" in _schema_types(prop_schema) and isinstance(prop_schema.get("enum"), list):
-                    raw_value = self.data.getlist(input_name)
-
-                if prop_schema.get("x-sensitive") and raw_value == "":
-                    continue
-
-                try:
-                    coerced_value = _coerce_plugin_config_value(raw_value, prop_schema)
-                except (TypeError, ValueError, json.JSONDecodeError) as err:
-                    self.add_error("config", forms.ValidationError(f"{config_key}: {err}"))
-                    continue
-                except forms.ValidationError as err:
-                    self.add_error("config", forms.ValidationError(f"{config_key}: {err.messages[0]}"))
-                    continue
-
-                base_value = effective_config.get(config_key, prop_schema.get("default", ""))
-                if _same_config_value(coerced_value, base_value):
-                    continue
-
-                existing_value = overrides.get(config_key)
-                if config_key in overrides and not _same_config_value(existing_value, coerced_value):
-                    self.add_error(
-                        "config",
-                        forms.ValidationError(
-                            f"{config_key} was set differently under {sources[config_key]} and {plugin_name}. Set it once in Custom config overrides.",
-                        ),
-                    )
-                    continue
-
-                overrides[config_key] = coerced_value
-                sources[config_key] = plugin_name
-
-        return overrides
-
-    def plugin_config_keys(self) -> set[str]:
-        return {
-            str(config_key)
-            for schema in discover_plugin_configs().values()
-            for config_key, prop_schema in (schema.get("properties") or {}).items()
-            if isinstance(prop_schema, dict)
-        }
-
-
-def get_plugin_config_binary_urls(runtime_config: Mapping[str, Any]) -> dict[str, str]:
-    from archivebox.config.views import get_environment_binary_url, get_installed_binary_change_url
-    from archivebox.machine.models import Binary, Machine
-
-    binary_keys = {
-        str(config_key)
-        for schema in discover_plugin_configs().values()
-        for config_key, prop_schema in (schema.get("properties") or {}).items()
-        if isinstance(prop_schema, dict) and str(config_key).endswith("_BINARY")
-    }
-    urls: dict[str, str] = {}
-    machine = Machine.current()
-    for key in binary_keys:
-        value = str(runtime_config.get(key) or "").strip()
-        if not value:
-            continue
-        name = Path(value).name if "/" in value else value
-        binary = Binary.objects.get_valid_binary(value, machine=machine)
-        if binary is None and "/" in value:
-            binary = (
-                Binary.objects.exclude(abspath="")
-                .exclude(abspath__isnull=True)
-                .filter(machine=machine, abspath=value)
-                .order_by("-modified_at")
-                .first()
-            )
-        if binary is None and name != value:
-            binary = Binary.objects.get_valid_binary(name, machine=machine)
-        urls[key] = get_installed_binary_change_url(getattr(binary, "name", name), binary) or get_environment_binary_url(name)
-    return urls
+def edit_string_for_tag_names(tags) -> str:
+    names = []
+    for tag in tags:
+        name = tag.name
+        names.append(f'"{name}"' if "," in name or " " in name else name)
+    return ", ".join(sorted(names))
 
 
 class AddLinkForm(PluginConfigFormMixin, forms.Form):
+    allow_crawl_execution_config_fields = False
+
     # Basic fields
     url = forms.CharField(
         label="URLs",
-        strip=True,
+        strip=False,
         widget=forms.Textarea(
             attrs={
                 "data-url-regex": URL_REGEX.pattern,
+                "placeholder": (
+                    "\n",
+                    "Enter URL(s) to archive. Any format is ok: one per line, CSV, JSON, embedded in text, etc."
+                    "Examples:\n\n"
+                    "https://example.com\n\n"
+                    "https://news.ycombinator.com,https://news.google.com\n\n"
+                    "Or any text-based content [containing URLs](https://github.com/ArchiveBox/ArchiveBox)...",
+                ),
             },
         ),
         required=True,
@@ -526,7 +125,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         widget=forms.RadioSelect(attrs={"class": "depth-selection"}),
     )
     max_urls = forms.IntegerField(
-        label="Max URLs",
+        label="Max crawl URLs",
         required=False,
         min_value=0,
         initial=0,
@@ -548,6 +147,29 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
+    crawl_timeout = forms.CharField(
+        label="Max crawl time",
+        required=False,
+        initial=0,
+        widget=forms.TextInput(
+            attrs={
+                "pattern": TIMEOUT_INPUT_PATTERN,
+                "title": "Use 0, integer seconds, or a duration like 1.5m or 1hr. Non-zero values must be greater than 10 seconds.",
+                "placeholder": "0, 300, 1.5m, or 1hr",
+            },
+        ),
+    )
+    timeout = forms.CharField(
+        label="Max subtask time",
+        required=False,
+        widget=forms.TextInput(
+            attrs={
+                "pattern": TIMEOUT_INPUT_PATTERN,
+                "title": "Use integer seconds or a duration like 1.5m or 1hr. Non-zero values must be greater than 10 seconds.",
+                "placeholder": "60, 1.5m, or 1hr",
+            },
+        ),
+    )
     snapshot_max_size = forms.CharField(
         label="Max snapshot size",
         required=False,
@@ -558,8 +180,18 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
+    delete_after = forms.CharField(
+        label="Delete after",
+        required=False,
+        initial="0",
+        widget=forms.TextInput(
+            attrs={
+                "placeholder": "0 = keep forever, or e.g. 1d / 6mo",
+            },
+        ),
+    )
     crawl_max_concurrent_snapshots = forms.IntegerField(
-        label="Max concurrent snapshots",
+        label="Max in parallel",
         required=False,
         min_value=1,
         widget=forms.NumberInput(
@@ -641,14 +273,20 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         ),
     )
     persona = forms.ModelChoiceField(
-        label="Persona (authentication profile)",
+        label="Persona (configuration profile)",
         required=False,
         queryset=Persona.objects.none(),
         empty_label=None,
         to_field_name="name",
     )
-    index_only = forms.BooleanField(
-        label="Index only dry run (add crawl but don't archive yet)",
+    permissions = forms.ChoiceField(
+        label="Permissions",
+        choices=PERMISSIONS_CHOICES,
+        initial="public",
+        required=True,
+    )
+    start_paused = forms.BooleanField(
+        label="Start paused",
         initial=False,
         required=False,
     )
@@ -660,21 +298,57 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
     )
 
     def __init__(self, *args, **kwargs):
+        self.request = kwargs.pop("request", None)
+        self.can_override_crawl_config = False
+        if self.request is not None:
+            user = self.request.user
+            self.can_override_crawl_config = bool(user.is_authenticated and user.is_active and user.is_superuser)
         super().__init__(*args, **kwargs)
 
         default_persona = Persona.get_or_create_default()
-        self.fields["persona"].queryset = Persona.objects.order_by("name")
-        self.fields["persona"].initial = default_persona.name
-        self.fields["crawl_max_concurrent_snapshots"].initial = get_config(persona=default_persona).CRAWL_MAX_CONCURRENT_SNAPSHOTS
+        persona_queryset = Persona.objects.order_by("name")
+        if not self.can_override_crawl_config:
+            persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
+        self.fields["persona"].queryset = persona_queryset
+
+        selected_persona = persona_queryset.filter(id=default_persona.id).first() or persona_queryset.first()
+        default_config = get_config(persona=selected_persona) if selected_persona else get_config()
+        if selected_persona:
+            self.fields["persona"].initial = selected_persona.name
+        self.fields["permissions"].initial = default_config.PERMISSIONS
+        self.fields["timeout"].initial = default_config.TIMEOUT
+        self.fields["crawl_max_concurrent_snapshots"].initial = default_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
+        self.fields["delete_after"].initial = default_config.DELETE_AFTER
+        self.fields["url_filters"].initial = {
+            "allowlist": "",
+            "denylist": "",
+            "same_domain_only": False,
+            "subpaths_only": False,
+            "only_new": bool(default_config.ONLY_NEW),
+        }
 
-        selected_persona = default_persona
         if self.is_bound:
-            selected_persona = Persona.objects.filter(name=str(self.data.get(self.add_prefix("persona")) or "")).first() or default_persona
-        self.build_plugin_groups(get_config(persona=selected_persona))
+            selected_persona = (
+                persona_queryset.filter(name=str(self.data.get(self.add_prefix("persona")) or "")).first() or selected_persona
+            )
+        if self.can_override_crawl_config:
+            self.build_plugin_groups(get_config(persona=selected_persona) if selected_persona else get_config())
+        else:
+            all_plugins = get_plugins()
+            for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
+                get_choice_field(self, field_name).choices = [(p, p) for p in all_plugins if p in plugin_names]
+            get_choice_field(self, "other_plugins").choices = [(p, p) for p in all_plugins]
+            self.plugin_groups = []
 
     def clean(self):
         cleaned_data = super().clean() or {}
 
+        if not self.can_override_crawl_config:
+            cleaned_data["plugins"] = []
+            cleaned_data["plugin_config"] = {}
+            cleaned_data["config"] = {}
+            return cleaned_data
+
         # Combine all plugin groups into single list
         all_selected_plugins = []
         for field in [
@@ -704,12 +378,9 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 
     def clean_url(self):
         value = self.cleaned_data.get("url") or ""
-        valid_urls = []
-        for url in find_all_urls(value):
-            valid_urls.append(url)
-        if not valid_urls:
+        if not list(find_all_urls(value)):
             raise forms.ValidationError("Enter at least one valid URL.")
-        return "\n".join(valid_urls)
+        return value
 
     def clean_url_filters(self):
         from archivebox.crawls.models import Crawl
@@ -719,6 +390,8 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
             "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
             "same_domain_only": bool(value.get("same_domain_only")),
+            "subpaths_only": bool(value.get("subpaths_only")),
+            "only_new": bool(value.get("only_new")),
         }
 
     def clean_max_urls(self):
@@ -737,6 +410,37 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             raise forms.ValidationError("Max crawl size must be 0 or a positive number of bytes.")
         return value
 
+    def clean_crawl_timeout(self):
+        return self._clean_timeout_seconds(self.cleaned_data.get("crawl_timeout"), "Max crawl time", blank_value=0)
+
+    def clean_timeout(self):
+        return self._clean_timeout_seconds(self.cleaned_data.get("timeout"), "Max subtask time", blank_value=None)
+
+    def _clean_timeout_seconds(self, raw_value, field_label: str, *, blank_value):
+        raw_value = str(raw_value or "").strip().lower()
+        if not raw_value:
+            return blank_value
+        if raw_value.isdigit():
+            value = int(raw_value)
+        else:
+            match = re.fullmatch(r"(\d+(?:\.\d+)?)\s*(s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours)", raw_value)
+            if not match:
+                raise forms.ValidationError(f"{field_label} must be seconds or a duration like 1.5m or 1hr.")
+            amount_str, unit = match.groups()
+            try:
+                amount = Decimal(amount_str)
+            except InvalidOperation as err:
+                raise forms.ValidationError(f"{field_label} must be seconds or a duration like 1.5m or 1hr.") from err
+            multiplier = 1
+            if unit in {"m", "min", "mins", "minute", "minutes"}:
+                multiplier = 60
+            elif unit in {"h", "hr", "hrs", "hour", "hours"}:
+                multiplier = 60 * 60
+            value = int((amount * multiplier).to_integral_value(rounding=ROUND_CEILING))
+        if 0 < value <= 10:
+            raise forms.ValidationError(f"{field_label} must be 0 or greater than 10 seconds.")
+        return value
+
     def clean_snapshot_max_size(self):
         raw_value = str(self.cleaned_data.get("snapshot_max_size") or "").strip()
         if not raw_value:
@@ -749,6 +453,14 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             raise forms.ValidationError("Max snapshot size must be 0 or a positive number of bytes.")
         return value
 
+    def clean_delete_after(self):
+        raw_value = str(self.cleaned_data.get("delete_after") or "0").strip() or "0"
+        try:
+            parse_delete_after(raw_value)
+        except ValueError as err:
+            raise forms.ValidationError(str(err))
+        return raw_value
+
     def clean_crawl_max_concurrent_snapshots(self):
         value = self.cleaned_data.get("crawl_max_concurrent_snapshots")
         if value in (None, ""):
@@ -774,7 +486,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 class TagWidget(forms.TextInput):
     def format_value(self, value):
         if value is not None and not isinstance(value, str):
-            value = edit_string_for_tags(value)
+            value = edit_string_for_tag_names(value)
         return super().format_value(value)
 
 
@@ -783,12 +495,7 @@ class TagField(forms.CharField):
 
     def clean(self, value):
         value = super().clean(value)
-        try:
-            return parse_tags(value)
-        except ValueError:
-            raise forms.ValidationError(
-                "Please provide a comma-separated list of tags.",
-            )
+        return parse_tag_string(value)
 
     def has_changed(self, initial, data):
         # Always return False if the field is disabled since self.bound_data

archivebox/core/host_utils.py

@@ -1,259 +0,0 @@
-from __future__ import annotations
-
-import re
-from typing import Any
-from urllib.parse import urlparse
-
-from archivebox.config.common import get_config
-
-
-_SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
-_SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P<suffix>[0-9a-fA-F]{12})$")
-
-
-def split_host_port(host: str) -> tuple[str, str | None]:
-    parsed = urlparse(f"//{host}")
-    hostname = (parsed.hostname or host or "").lower()
-    port = str(parsed.port) if parsed.port else None
-    return hostname, port
-
-
-def _normalize_base_url(value: str | None) -> str:
-    if not value:
-        return ""
-    base = value.strip()
-    if not base:
-        return ""
-    if "://" not in base:
-        base = f"http://{base}"
-    parsed = urlparse(base)
-    if not parsed.netloc:
-        return ""
-    return f"{parsed.scheme}://{parsed.netloc}"
-
-
-def normalize_base_url(value: str | None) -> str:
-    return _normalize_base_url(value)
-
-
-def get_listen_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    return (config.LISTEN_HOST or "").strip()
-
-
-def get_listen_parts(config: dict[str, Any] | None = None, **config_kwargs: Any) -> tuple[str, str | None]:
-    config = config or get_config(**config_kwargs)
-    return split_host_port(get_listen_host(config=config))
-
-
-def _build_listen_host(subdomain: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    host, port = get_listen_parts(config=config)
-    if not host:
-        return ""
-    full_host = f"{subdomain}.{host}" if subdomain else host
-    if port:
-        return f"{full_host}:{port}"
-    return full_host
-
-
-def get_admin_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    override = _normalize_base_url(config.ADMIN_BASE_URL)
-    if override:
-        return urlparse(override).netloc.lower()
-    return _build_listen_host("admin", config=config)
-
-
-def get_web_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    override = _normalize_base_url(config.ARCHIVE_BASE_URL)
-    if override:
-        return urlparse(override).netloc.lower()
-    return _build_listen_host("web", config=config)
-
-
-def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    return _build_listen_host("api", config=config)
-
-
-def get_public_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    return _build_listen_host("public", config=config)
-
-
-def get_snapshot_subdomain(snapshot_id: str) -> str:
-    normalized = re.sub(r"[^0-9a-fA-F]", "", snapshot_id or "")
-    suffix = (normalized[-12:] if len(normalized) >= 12 else normalized).lower()
-    return f"snap-{suffix}"
-
-
-def get_snapshot_host(snapshot_id: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    return _build_listen_host(get_snapshot_subdomain(snapshot_id), config=config)
-
-
-def get_original_host(domain: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    return _build_listen_host(domain, config=config)
-
-
-def is_snapshot_subdomain(subdomain: str) -> bool:
-    value = (subdomain or "").strip()
-    return bool(_SNAPSHOT_SUBDOMAIN_RE.match(value) or _SNAPSHOT_ID_RE.match(value))
-
-
-def get_snapshot_lookup_key(snapshot_ref: str) -> str:
-    value = (snapshot_ref or "").strip().lower()
-    match = _SNAPSHOT_SUBDOMAIN_RE.match(value)
-    if match:
-        return match.group("suffix")
-    return value
-
-
-def get_listen_subdomain(request_host: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return ""
-    req_host, req_port = split_host_port(request_host)
-    listen_host, listen_port = get_listen_parts(config=config)
-    if not listen_host:
-        return ""
-    if listen_port and req_port and listen_port != req_port:
-        return ""
-    if req_host == listen_host:
-        return ""
-    suffix = f".{listen_host}"
-    if req_host.endswith(suffix):
-        return req_host[: -len(suffix)]
-    return ""
-
-
-def host_matches(request_host: str, target_host: str) -> bool:
-    if not request_host or not target_host:
-        return False
-    req_host, req_port = split_host_port(request_host)
-    target_host_only, target_port = split_host_port(target_host)
-    if req_host != target_host_only:
-        return False
-    if target_port and req_port and target_port != req_port:
-        return False
-    return True
-
-
-def _scheme_from_request(request=None, config: dict[str, Any] | None = None) -> str:
-    if request and request.scheme != "http":
-        return request.scheme
-    config = config or get_config()
-    for base_url in (config.ARCHIVE_BASE_URL, config.ADMIN_BASE_URL):
-        override = _normalize_base_url(base_url)
-        if override:
-            return urlparse(override).scheme
-    if request:
-        return request.scheme
-    return "http"
-
-
-def _build_base_url_for_host(host: str, request=None, config: dict[str, Any] | None = None) -> str:
-    if not host:
-        return ""
-    scheme = _scheme_from_request(request, config=config)
-    return f"{scheme}://{host}"
-
-
-def get_admin_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    override = _normalize_base_url(config.ADMIN_BASE_URL)
-    if override:
-        return override
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_base_url_for_host(get_listen_host(config=config), request=request, config=config)
-    return _build_base_url_for_host(get_admin_host(config=config), request=request, config=config)
-
-
-def get_web_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    override = _normalize_base_url(config.ARCHIVE_BASE_URL)
-    if override:
-        return override
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_base_url_for_host(get_listen_host(config=config), request=request, config=config)
-    return _build_base_url_for_host(get_web_host(config=config), request=request, config=config)
-
-
-def get_api_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_base_url_for_host(get_listen_host(config=config), request=request, config=config)
-    return _build_base_url_for_host(get_api_host(config=config), request=request, config=config)
-
-
-def get_public_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    return _build_base_url_for_host(get_public_host(config=config), request=request, config=config)
-
-
-# Backwards-compat aliases (archive == web)
-def get_archive_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return get_web_base_url(request=request, config=config, **config_kwargs)
-
-
-def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_url(get_web_base_url(request=request, config=config), f"/snapshot/{snapshot_id}")
-    return _build_base_url_for_host(get_snapshot_host(snapshot_id, config=config), request=request, config=config)
-
-
-def get_original_base_url(domain: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_url(get_web_base_url(request=request, config=config), f"/original/{domain}")
-    return _build_base_url_for_host(get_original_host(domain, config=config), request=request, config=config)
-
-
-def build_admin_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_admin_base_url(request, config=config, **config_kwargs), path)
-
-
-def build_web_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_web_base_url(request, config=config, **config_kwargs), path)
-
-
-def build_api_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_api_base_url(request, config=config, **config_kwargs), path)
-
-
-def build_archive_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_archive_base_url(request, config=config, **config_kwargs), path)
-
-
-def build_snapshot_url(snapshot_id: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_snapshot_base_url(snapshot_id, request=request, config=config, **config_kwargs), path)
-
-
-def build_original_url(domain: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_original_base_url(domain, request=request, config=config, **config_kwargs), path)
-
-
-def _build_url(base_url: str, path: str) -> str:
-    if not base_url:
-        if not path:
-            return ""
-        return path if path.startswith("/") else f"/{path}"
-    if not path:
-        return base_url
-    return f"{base_url}{path if path.startswith('/') else f'/{path}'}"

archivebox/core/middleware.py

@@ -3,6 +3,7 @@ __package__ = "archivebox.core"
 import ipaddress
 import re
 from pathlib import Path
+from django.conf import settings
 from django.utils import timezone
 from django.contrib.auth.middleware import RemoteUserMiddleware
 from django.contrib.auth.models import AnonymousUser
@@ -15,15 +16,15 @@ from django.http import HttpResponseForbidden, HttpResponseNotModified
 from archivebox.config.common import get_config
 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH
-from archivebox.core.host_utils import (
+from archivebox.core.routes_util import (
     build_snapshot_url,
     build_admin_url,
     build_web_url,
     get_api_host,
     get_admin_host,
+    get_base_host,
     get_listen_host,
     get_listen_subdomain,
-    get_public_host,
     get_web_host,
     host_matches,
     is_snapshot_subdomain,
@@ -36,14 +37,32 @@ ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in"
 
 
 def _admin_login_hint_cookie_domain(config) -> str | None:
+    """Resolve the parent domain to scope the cross-subdomain login hint.
+
+    NOTE: this cookie carries only the single bit "user is logged in on
+    admin somewhere"; it MUST NOT be confused with the session cookie,
+    which stays admin-host-scoped (see core/settings.py
+    SESSION_COOKIE_DOMAIN comment — admin/web is a security boundary).
+
+    Returns the hostname portion of ``get_base_host`` (which respects
+    ``BASE_URL`` and falls back to the local-bind mapping). Strips the
+    port — cookie ``Domain=`` attributes don't include ports. Returns
+    ``None`` when subdomain routing is off, the base host is empty, or
+    the base host is an IP / bare ``localhost`` (browsers reject
+    cross-host cookies for those).
+    """
     if not config.USES_SUBDOMAIN_ROUTING:
         return None
-    listen_host, _listen_port = split_host_port(get_listen_host(config=config))
+    base_host = get_base_host(config=config)
+    if not base_host:
+        return None
+    hostname, _port = split_host_port(base_host)
+    if not hostname or hostname == "localhost":
+        return None
     try:
-        ipaddress.ip_address(listen_host)
+        ipaddress.ip_address(hostname)
     except ValueError:
-        if listen_host and listen_host != "localhost":
-            return listen_host
+        return hostname
     return None
 
 
@@ -66,6 +85,34 @@ def TimezoneMiddleware(get_response):
     return middleware
 
 
+def AdminCookieIsolationMiddleware(get_response):
+    def middleware(request):
+        response = get_response(request)
+
+        config = request.__dict__.get("archivebox_config")
+        if config is None:
+            config = get_config(resolve_plugins=False)
+            request.archivebox_config = config
+        if not config.USES_SUBDOMAIN_ROUTING:
+            return response
+
+        request_host = (request.get_host() or "").lower()
+        if host_matches(request_host, get_admin_host(config=config)):
+            return response
+
+        if host_matches(request_host, get_web_host(config=config)):
+            for cookie_name in tuple(response.cookies.keys()):
+                if cookie_name != ADMIN_LOGIN_HINT_COOKIE:
+                    response.cookies.pop(cookie_name, None)
+            return response
+
+        response.cookies.pop(settings.SESSION_COOKIE_NAME, None)
+        response.cookies.pop(settings.CSRF_COOKIE_NAME, None)
+        return response
+
+    return middleware
+
+
 def CacheControlMiddleware(get_response):
     snapshot_path_re = re.compile(r"^/[^/]+/\\d{8}/[^/]+/[0-9a-fA-F-]{8,36}/")
     static_cache_key = (get_COMMIT_HASH() or VERSION or "dev").strip()
@@ -100,11 +147,11 @@ def CacheControlMiddleware(get_response):
 
         if "/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path):
             if not response.get("Cache-Control"):
-                config = getattr(request, "archivebox_config", None)
+                config = request.__dict__.get("archivebox_config")
                 if config is None:
                     config = get_config(resolve_plugins=False)
                     request.archivebox_config = config
-                policy = "public" if config.PUBLIC_SNAPSHOTS else "private"
+                policy = "private" if config.PERMISSIONS == "private" else "public"
                 response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
                 # print('Set Cache-Control header to', response['Cache-Control'])
         return response
@@ -117,7 +164,7 @@ def ServerSecurityModeMiddleware(get_response):
     allowed_methods = {"GET", "HEAD", "OPTIONS"}
 
     def middleware(request):
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
         if config is None:
             config = get_config(resolve_plugins=False)
             request.archivebox_config = config
@@ -146,15 +193,17 @@ def HostRoutingMiddleware(get_response):
     )
 
     def middleware(request):
+        if request.path in {"/health", "/health/"}:
+            return get_response(request)
+
         request_host = (request.get_host() or "").lower()
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
         if config is None:
             config = get_config(resolve_plugins=False)
             request.archivebox_config = config
         admin_host = get_admin_host(config=config)
         web_host = get_web_host(config=config)
         api_host = get_api_host(config=config)
-        public_host = get_public_host(config=config)
         listen_host = get_listen_host(config=config)
         subdomain = get_listen_subdomain(request_host, config=config)
 
@@ -163,18 +212,34 @@ def HostRoutingMiddleware(get_response):
         if request.path.startswith("/static/") or request.path in {"/favicon.ico", "/robots.txt"}:
             return get_response(request)
 
-        if config.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host):
+        if config.USES_SUBDOMAIN_ROUTING and config.BASE_URL and not host_matches(request_host, admin_host):
+            add_should_redirect = not config.PUBLIC_ADD_VIEW and (request.path == "/add" or request.path.startswith("/add/"))
             if (
                 request.path == "/admin"
                 or request.path.startswith("/admin/")
                 or request.path == "/accounts"
                 or request.path.startswith("/accounts/")
+                or add_should_redirect
             ):
                 target = build_admin_url(request.path, request=request)
                 if request.META.get("QUERY_STRING"):
                     target = f"{target}?{request.META['QUERY_STRING']}"
                 return redirect(target)
 
+        if subdomain and is_snapshot_subdomain(subdomain):
+            view = SnapshotHostView.as_view()
+            return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
+
+        # In subdomain mode with no explicit BASE_URL we can't safely emit
+        # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
+        # request's own Host (via the request-host fallback in get_base_url),
+        # so prepending ``admin.`` to whatever the client sent produces a
+        # redirect chain of ``admin.admin.admin.<host>``. Pass the request
+        # through; the misconfig banner on the rendered page tells the user
+        # to pin BASE_URL so the redirects can resume.
+        if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
+            return get_response(request)
+
         if not config.USES_SUBDOMAIN_ROUTING:
             if host_matches(request_host, listen_host):
                 return get_response(request)
@@ -240,16 +305,7 @@ def HostRoutingMiddleware(get_response):
             request._cached_user = request.user
             return get_response(request)
 
-        if host_matches(request_host, public_host):
-            if request.COOKIES.get(ADMIN_LOGIN_HINT_COOKIE) == "1" and (request.path == "/public" or request.path.startswith("/public/")):
-                target = build_admin_url("/admin/core/snapshot/", request=request)
-                return redirect(target)
-            return get_response(request)
-
         if subdomain:
-            if is_snapshot_subdomain(subdomain):
-                view = SnapshotHostView.as_view()
-                return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
             view = OriginalDomainHostView.as_view()
             return view(request, domain=subdomain, path=request.path.lstrip("/"))
 
@@ -259,7 +315,13 @@ def HostRoutingMiddleware(get_response):
                 target = f"{target}?{request.META['QUERY_STRING']}"
             return redirect(target)
 
-        if admin_host or web_host:
+        if (admin_host or web_host) and config.BASE_URL:
+            # Only force a canonical-host redirect when BASE_URL was set
+            # explicitly. If BASE_URL is empty (e.g. 0.7.3 → 0.9.0 upgrade
+            # where the user has CSRF_TRUSTED_ORIGINS but never set BASE_URL),
+            # the subdomain we'd redirect to may not actually resolve in the
+            # user's reverse proxy — serve the request as-is instead and let
+            # the misconfig banner surface the problem in the page.
             target = build_web_url(request.path, request=request)
             if target:
                 if request.META.get("QUERY_STRING"):
@@ -275,7 +337,7 @@ class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
     header = "HTTP_REMOTE_USER"
 
     def process_request(self, request):
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
         if config is None:
             config = get_config(resolve_plugins=False)
             request.archivebox_config = config

archivebox/core/migrations/0023_upgrade_to_0_9_0.py

@@ -9,6 +9,9 @@ from datetime import datetime
 import json
 
 
+PROGRESS_EVERY = 10000
+
+
 def get_table_columns(table_name):
     """Get list of column names for a table."""
     cursor = connection.cursor()
@@ -64,7 +67,9 @@ def upgrade_core_tables(apps, schema_editor):
     has_uuid = "uuid" in archiveresult_cols
     has_abid = "abid" in archiveresult_cols
 
-    print(f"DEBUG: ArchiveResult row_count={row_count}, has_data={has_data}, has_uuid={has_uuid}, has_abid={has_abid}")
+    if has_data:
+        source_schema = "0.8.x abid" if has_abid and not has_uuid else "0.8.x uuid" if has_uuid else "0.7.x"
+        print(f"    - Rebuilding core tables from {source_schema} schema ({row_count} ArchiveResults)...")
 
     # ============================================================================
     # PART 1: Upgrade core_archiveresult table
@@ -113,7 +118,7 @@ def upgrade_core_tables(apps, schema_editor):
 
         if has_uuid and not has_abid:
             # Migrating from v0.7.2+ (has uuid column)
-            print("Migrating ArchiveResult from v0.7.2+ schema (with uuid)...")
+            print(f"      copying {row_count} ArchiveResults...")
             select_cols = ["id", "uuid", "snapshot_id", "cmd", "pwd", "cmd_version", "start_ts", "end_ts", "status", "extractor", "output"]
             if has_archiveresult_created_at:
                 select_cols.append("created_at")
@@ -121,7 +126,7 @@ def upgrade_core_tables(apps, schema_editor):
                 select_cols.append("modified_at")
             cursor.execute(f"SELECT {', '.join(select_cols)} FROM core_archiveresult")
             old_records = cursor.fetchall()
-            for record in old_records:
+            for i, record in enumerate(old_records, start=1):
                 values = dict(zip(select_cols, record))
                 try:
                     new_uuid = UUID(str(values["uuid"])).hex
@@ -153,12 +158,14 @@ def upgrade_core_tables(apps, schema_editor):
                         values.get("modified_at") or end_ts,
                     ),
                 )
+                if i % PROGRESS_EVERY == 0:
+                    print(f"        copied {i}/{len(old_records)} ArchiveResults...")
         elif has_abid and not has_uuid:
             # Migrating from v0.8.6rc0 (has abid instead of uuid)
-            print("Migrating ArchiveResult from v0.8.6rc0 schema...")
+            print(f"      copying {row_count} ArchiveResults...")
             cursor.execute(f"SELECT {', '.join(archiveresult_select_cols)} FROM core_archiveresult")
             old_records = cursor.fetchall()
-            for record in old_records:
+            for i, record in enumerate(old_records, start=1):
                 values = dict(zip(archiveresult_select_cols, record))
                 try:
                     new_uuid = UUID(str(values["id"])).hex
@@ -189,12 +196,14 @@ def upgrade_core_tables(apps, schema_editor):
                         values.get("modified_at") or end_ts,
                     ),
                 )
+                if i % PROGRESS_EVERY == 0:
+                    print(f"        copied {i}/{len(old_records)} ArchiveResults...")
         else:
             # Migrating from v0.7.2 (no uuid or abid column - generate fresh UUIDs)
-            print("Migrating ArchiveResult from v0.7.2 schema (no uuid - generating UUIDs)...")
+            print(f"      copying {row_count} ArchiveResults...")
             cursor.execute(f"SELECT {', '.join(archiveresult_select_cols)} FROM core_archiveresult")
             old_records = cursor.fetchall()
-            for record in old_records:
+            for i, record in enumerate(old_records, start=1):
                 values = dict(zip(archiveresult_select_cols, record))
                 new_uuid = uuid7().hex
                 start_ts = values["start_ts"] or datetime.now().isoformat()
@@ -223,6 +232,9 @@ def upgrade_core_tables(apps, schema_editor):
                         values.get("modified_at") or end_ts,
                     ),
                 )
+                if i % PROGRESS_EVERY == 0:
+                    print(f"        copied {i}/{len(old_records)} ArchiveResults...")
+        print(f"      copied {len(old_records)} ArchiveResults")
 
     cursor.execute("DROP TABLE IF EXISTS core_archiveresult;")
     cursor.execute("ALTER TABLE core_archiveresult_new RENAME TO core_archiveresult;")
@@ -279,7 +291,7 @@ def upgrade_core_tables(apps, schema_editor):
 
             if has_added and not has_bookmarked_at:
                 # Migrating from v0.7.2 (has added/updated fields)
-                print("Migrating Snapshot from v0.7.2 schema...")
+                print("      copying Snapshots from 0.7.x schema...")
                 # timestamp is the legacy bookmark/import timestamp and archive/{timestamp} identity.
                 # added is the DB row creation/import time, and updated was renamed to downloaded_at in 0.8.x.
                 cursor.execute("""
@@ -301,12 +313,26 @@ def upgrade_core_tables(apps, schema_editor):
                         COALESCE(added, CURRENT_TIMESTAMP) as created_at,
                         COALESCE(updated, added, CURRENT_TIMESTAMP) as modified_at,
                         updated as downloaded_at,
-                        'queued' as status
+                        CASE
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                                AND core_archiveresult.status IN ('queued', 'started', 'backoff')
+                            )
+                            THEN 'queued'
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                            )
+                            THEN 'sealed'
+                            ELSE 'queued'
+                        END as status
                     FROM core_snapshot;
                 """)
+                print(f"      copied {cursor.rowcount} Snapshots")
             elif has_bookmarked_at and not has_added:
                 # Migrating from v0.8.6rc0 (already has bookmarked_at/created_at/modified_at)
-                print("Migrating Snapshot from v0.8.6rc0 schema...")
+                print("      copying Snapshots from 0.8.x schema...")
                 # Check what fields exist
                 has_status = "status" in snapshot_cols
                 has_retry_at = "retry_at" in snapshot_cols
@@ -320,7 +346,25 @@ def upgrade_core_tables(apps, schema_editor):
                     select_cols.append("REPLACE(crawl_id, '-', '')")
                 if has_status:
                     insert_cols.append("status")
-                    select_cols.append("status")
+                    select_cols.append(
+                        """
+                        CASE
+                            WHEN status IN ('sealed', 'started', 'paused') THEN status
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                                AND core_archiveresult.status IN ('queued', 'started', 'backoff')
+                            )
+                            THEN status
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                            )
+                            THEN 'sealed'
+                            ELSE status
+                        END
+                        """,
+                    )
                 if has_retry_at:
                     insert_cols.append("retry_at")
                     select_cols.append("retry_at")
@@ -330,6 +374,7 @@ def upgrade_core_tables(apps, schema_editor):
                     SELECT {", ".join(select_cols)}
                     FROM core_snapshot;
                 """)
+                print(f"      copied {cursor.rowcount} Snapshots")
             else:
                 print(f"Warning: Unexpected Snapshot schema - has_added={has_added}, has_bookmarked_at={has_bookmarked_at}")
 
@@ -380,7 +425,7 @@ def upgrade_core_tables(apps, schema_editor):
 
             if tag_id_type and "char" in tag_id_type.lower():
                 # v0.8.6rc0: Tag IDs are UUIDs, need to convert to INTEGER
-                print("Converting Tag IDs from UUID to INTEGER...")
+                print("      converting Tag IDs from UUID to integers...")
 
                 # Get all tags with their UUIDs
                 cursor.execute("SELECT id, name, slug, created_at, modified_at, created_by_id FROM core_tag ORDER BY name")
@@ -399,6 +444,8 @@ def upgrade_core_tables(apps, schema_editor):
                     """,
                         (i, name, slug, created_at, modified_at, created_by_id),
                     )
+                    if i % PROGRESS_EVERY == 0:
+                        print(f"        copied {i}/{len(tags)} Tags...")
 
                 # Update snapshot_tags to use new INTEGER IDs
                 cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_snapshot_tags'")
@@ -410,7 +457,7 @@ def upgrade_core_tables(apps, schema_editor):
                     cursor.execute("DELETE FROM core_snapshot_tags")
 
                     # Re-insert with new integer tag IDs
-                    for st_id, snapshot_id, old_tag_id in snapshot_tags:
+                    for i, (st_id, snapshot_id, old_tag_id) in enumerate(snapshot_tags, start=1):
                         new_tag_id = uuid_to_int_map.get(old_tag_id)
                         if new_tag_id:
                             cursor.execute(
@@ -420,14 +467,18 @@ def upgrade_core_tables(apps, schema_editor):
                             """,
                                 (st_id, snapshot_id, new_tag_id),
                             )
+                        if i % PROGRESS_EVERY == 0:
+                            print(f"        copied {i}/{len(snapshot_tags)} SnapshotTag rows...")
+                print(f"      copied {len(tags)} Tags")
             else:
                 # v0.7.2: Tag IDs are already INTEGER
-                print("Migrating Tag from v0.7.2 schema...")
+                print("      copying Tags from 0.7.x schema...")
                 cursor.execute("""
                     INSERT OR IGNORE INTO core_tag_new (id, name, slug)
                     SELECT id, name, slug
                     FROM core_tag;
                 """)
+                print(f"      copied {cursor.rowcount} Tags")
 
     cursor.execute("DROP TABLE IF EXISTS core_tag;")
     cursor.execute("ALTER TABLE core_tag_new RENAME TO core_tag;")
@@ -437,7 +488,7 @@ def upgrade_core_tables(apps, schema_editor):
     cursor.execute("CREATE INDEX IF NOT EXISTS core_tag_created_by_id_idx ON core_tag(created_by_id);")
 
     if has_data:
-        print("✓ Core tables upgraded to v0.9.0")
+        print("    ✓ Core table rebuild complete")
 
 
 class Migration(migrations.Migration):

archivebox/core/migrations/0025_alter_archiveresult_options_alter_snapshot_options_and_more.py

@@ -40,12 +40,7 @@ def copy_old_fields_to_new(apps, schema_editor):
 
     # NOTE: Snapshot timestamps (added→bookmarked_at, updated→modified_at) were already
     # transformed by migration 0023, so we don't need to copy them here.
-    # NOTE: UUIDs are already populated by migration 0023 for all migration paths
-
-    # Debug: Check Snapshot timestamps at end of RunPython
-    cursor.execute("SELECT id, bookmarked_at, modified_at FROM core_snapshot LIMIT 2")
-    snap_after = cursor.fetchall()
-    print(f"DEBUG 0025: Snapshot timestamps at END of RunPython: {snap_after}")
+    # NOTE: UUIDs are already populated by migration 0023 for all migration paths.
 
 
 class Migration(migrations.Migration):
@@ -151,70 +146,77 @@ class Migration(migrations.Migration):
             field=models.DateTimeField(blank=True, db_index=True, default=django.utils.timezone.now, null=True),
         ),
         # NOTE: bookmarked_at, created_at, and downloaded_at already added by migration 0023
-        migrations.AddField(
-            model_name="snapshot",
-            name="config",
-            field=models.JSONField(default=dict),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="current_step",
-            field=models.PositiveSmallIntegerField(
-                db_index=True,
-                default=0,
-                help_text="Current hook step being executed (0-9). Used for sequential hook execution.",
-            ),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="depth",
-            field=models.PositiveSmallIntegerField(db_index=True, default=0),
-        ),
-        # NOTE: downloaded_at already added by migration 0023 so it can preserve
-        # v0.7.x updated / v0.8.x downloaded_at values without a duplicate table rebuild.
-        # NOTE: fs_version already added by migration 0023 with default='0.8.0'
-        # NOTE: modified_at already added by migration 0023
-        migrations.AddField(
-            model_name="snapshot",
-            name="notes",
-            field=models.TextField(blank=True, default=""),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="num_uses_failed",
-            field=models.PositiveIntegerField(default=0),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="num_uses_succeeded",
-            field=models.PositiveIntegerField(default=0),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="parent_snapshot",
-            field=models.ForeignKey(
-                blank=True,
-                help_text="Parent snapshot that discovered this URL (for recursive crawling)",
-                null=True,
-                on_delete=django.db.models.deletion.SET_NULL,
-                related_name="child_snapshots",
-                to="core.snapshot",
-            ),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="retry_at",
-            field=models.DateTimeField(blank=True, db_index=True, default=django.utils.timezone.now, null=True),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="status",
-            field=models.CharField(
-                choices=[("queued", "Queued"), ("started", "Started"), ("sealed", "Sealed")],
-                db_index=True,
-                default="queued",
-                max_length=15,
-            ),
+        # 0023 rebuilds core_snapshot with these columns already present so it
+        # can preserve legacy status/retry_at/config data in one tight SQLite
+        # table copy. Keep this migration state-only: running normal AddField
+        # operations here causes SQLite to rebuild the table from the pre-0025
+        # state and repopulate existing columns with defaults, reopening sealed
+        # migrated rows as queued.
+        migrations.SeparateDatabaseAndState(
+            database_operations=[],
+            state_operations=[
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="config",
+                    field=models.JSONField(default=dict),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="current_step",
+                    field=models.PositiveSmallIntegerField(
+                        db_index=True,
+                        default=0,
+                        help_text="Current hook step being executed (0-9). Used for sequential hook execution.",
+                    ),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="depth",
+                    field=models.PositiveSmallIntegerField(db_index=True, default=0),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="notes",
+                    field=models.TextField(blank=True, default=""),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="num_uses_failed",
+                    field=models.PositiveIntegerField(default=0),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="num_uses_succeeded",
+                    field=models.PositiveIntegerField(default=0),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="parent_snapshot",
+                    field=models.ForeignKey(
+                        blank=True,
+                        help_text="Parent snapshot that discovered this URL (for recursive crawling)",
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="child_snapshots",
+                        to="core.snapshot",
+                    ),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="retry_at",
+                    field=models.DateTimeField(blank=True, db_index=True, default=django.utils.timezone.now, null=True),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="status",
+                    field=models.CharField(
+                        choices=[("queued", "Queued"), ("started", "Started"), ("sealed", "Sealed")],
+                        db_index=True,
+                        default="queued",
+                        max_length=15,
+                    ),
+                ),
+            ],
         ),
         migrations.AddField(
             model_name="tag",

archivebox/core/migrations/0027_copy_archiveresult_to_process.py

@@ -7,6 +7,9 @@ from pathlib import Path
 from archivebox.uuid_compat import uuid7
 
 
+PROGRESS_EVERY = 10000
+
+
 def parse_cmd_field(cmd_raw):
     """
     Parse cmd field which could be:
@@ -140,35 +143,33 @@ def get_or_create_binary(cursor, machine_id, name, abspath, version):
     cursor.execute("PRAGMA table_info(machine_binary)")
     binary_cols = {row[1] for row in cursor.fetchall()}
 
-    # Use only columns that exist in current schema
-    # 0.8.x schema: id, created_at, modified_at, machine_id, name, binprovider, abspath, version, sha256, num_uses_failed, num_uses_succeeded
-    # 0.9.x schema adds: binproviders, overrides, status, retry_at, output_dir
-    if "binproviders" in binary_cols:
-        # 0.9.x schema
-        cursor.execute(
-            """
-            INSERT INTO machine_binary (
-                id, created_at, modified_at, machine_id,
-                name, binproviders, overrides, binprovider, abspath, version, sha256,
-                status, retry_at, output_dir,
-                num_uses_failed, num_uses_succeeded
-            ) VALUES (?, ?, ?, ?, ?, 'env', '{}', 'env', ?, ?, '',
-                      'succeeded', NULL, '', 0, 0)
+    values_by_col = {
+        "id": binary_id,
+        "created_at": now,
+        "modified_at": now,
+        "machine_id": machine_id,
+        "name": name,
+        "binproviders": "env",
+        "overrides": "{}",
+        "binprovider": "env",
+        "abspath": abspath,
+        "version": version,
+        "sha256": "",
+        "status": "installed",
+        "retry_at": None,
+        "output_dir": "",
+        "num_uses_failed": 0,
+        "num_uses_succeeded": 0,
+    }
+    insert_cols = [col for col in values_by_col if col in binary_cols]
+    placeholders = ", ".join(["?"] * len(insert_cols))
+    cursor.execute(
+        f"""
+        INSERT INTO machine_binary ({", ".join(insert_cols)})
+        VALUES ({placeholders})
         """,
-            [binary_id, now, now, machine_id, name, abspath, version],
-        )
-    else:
-        # 0.8.x schema (simpler)
-        cursor.execute(
-            """
-            INSERT INTO machine_binary (
-                id, created_at, modified_at, machine_id,
-                name, binprovider, abspath, version, sha256,
-                num_uses_failed, num_uses_succeeded
-            ) VALUES (?, ?, ?, ?, ?, 'env', ?, ?, '', 0, 0)
-        """,
-            [binary_id, now, now, machine_id, name, abspath, version],
-        )
+        [values_by_col[col] for col in insert_cols],
+    )
 
     return binary_id
 
@@ -214,36 +215,42 @@ def create_process(cursor, machine_id, pwd, cmd, status, exit_code, started_at,
     # Set retry_at to now for queued processes, NULL otherwise
     retry_at = now if status == "queued" else None
 
+    cursor.execute("PRAGMA table_info(machine_process)")
+    process_cols = {row[1] for row in cursor.fetchall()}
+    values_by_col = {
+        "id": process_id,
+        "created_at": now,
+        "modified_at": now,
+        "machine_id": machine_id,
+        "parent_id": None,
+        "process_type": "hook",
+        "worker_type": "",
+        "pwd": pwd,
+        "cmd": cmd_json,
+        "env": "{}",
+        "timeout": 120,
+        "pid": None,
+        "exit_code": exit_code,
+        "stdout": "",
+        "stderr": "",
+        "started_at": started_at,
+        "ended_at": ended_at,
+        "binary_id": binary_id,
+        "iface_id": None,
+        "url": None,
+        "status": status,
+        "retry_at": retry_at,
+        "num_uses_failed": 0,
+        "num_uses_succeeded": 0,
+    }
+    insert_cols = [col for col in values_by_col if col in process_cols]
+    placeholders = ", ".join(["?"] * len(insert_cols))
     cursor.execute(
-        """
-        INSERT INTO machine_process (
-            id, created_at, modified_at, machine_id, parent_id, process_type,
-            pwd, cmd, env, timeout,
-            pid, exit_code, stdout, stderr,
-            started_at, ended_at,
-            binary_id, iface_id, url,
-            status, retry_at
-        ) VALUES (?, ?, ?, ?, NULL, 'cli',
-                  ?, ?, '{}', 120,
-                  NULL, ?, '', '',
-                  ?, ?,
-                  ?, NULL, NULL,
-                  ?, ?)
-    """,
-        [
-            process_id,
-            now,
-            now,
-            machine_id,
-            pwd,
-            cmd_json,
-            exit_code,
-            started_at,
-            ended_at,
-            binary_id,
-            status,
-            retry_at,
-        ],
+        f"""
+        INSERT INTO machine_process ({", ".join(insert_cols)})
+        VALUES ({placeholders})
+        """,
+        [values_by_col[col] for col in insert_cols],
     )
 
     return process_id
@@ -274,13 +281,8 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
     cursor.execute("PRAGMA table_info(core_archiveresult)")
     cols = {row[1] for row in cursor.fetchall()}
 
-    print(f"DEBUG 0027: Columns found: {sorted(cols)}")
-    print(
-        f"DEBUG 0027: Has cmd={('cmd' in cols)}, pwd={('pwd' in cols)}, cmd_version={('cmd_version' in cols)}, process_id={('process_id' in cols)}",
-    )
-
     if "cmd" not in cols or "pwd" not in cols or "cmd_version" not in cols:
-        print("✓ Fresh install or fields already removed - skipping data copy")
+        print("    ✓ ArchiveResult process metadata already migrated")
         return
 
     # Check if process_id field exists (should exist from 0026)
@@ -304,10 +306,10 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
     results = cursor.fetchall()
 
     if not results:
-        print("✓ No ArchiveResults need Process migration")
+        print("    ✓ No ArchiveResults need Process migration")
         return
 
-    print(f"Migrating {len(results)} ArchiveResults to Process records...")
+    print(f"    - Migrating {len(results)} ArchiveResults to Process rows...")
 
     migrated_count = 0
     skipped_count = 0
@@ -316,16 +318,10 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
     for i, row in enumerate(results):
         ar_id, snapshot_id, plugin, cmd_raw, pwd, cmd_version, status, start_ts, end_ts, created_at = row
 
-        if i == 0:
-            print(f"DEBUG 0027: First row: ar_id={ar_id}, plugin={plugin}, cmd={cmd_raw[:50] if cmd_raw else None}, status={status}")
-
         try:
             # Parse cmd field
             cmd_array = parse_cmd_field(cmd_raw)
 
-            if i == 0:
-                print(f"DEBUG 0027: Parsed cmd: {cmd_array}")
-
             # Extract binary info from cmd[0] if available
             binary_id = None
             if cmd_array and cmd_array[0]:
@@ -342,9 +338,6 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
                     binary_version,
                 )
 
-                if i == 0:
-                    print(f"DEBUG 0027: Created Binary: id={binary_id}, name={binary_name}")
-
             # Map status
             process_status, exit_code = map_status(status)
 
@@ -365,9 +358,6 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
                 binary_id=binary_id,
             )
 
-            if i == 0:
-                print(f"DEBUG 0027: Created Process: id={process_id}")
-
             # Link ArchiveResult to Process
             cursor.execute(
                 "UPDATE core_archiveresult SET process_id = ? WHERE id = ?",
@@ -375,9 +365,8 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
             )
 
             migrated_count += 1
-
-            if i == 0:
-                print("DEBUG 0027: Linked ArchiveResult to Process")
+            if migrated_count % PROGRESS_EVERY == 0:
+                print(f"      migrated {migrated_count}/{len(results)} ArchiveResults...")
 
         except Exception as e:
             print(f"✗ Error migrating ArchiveResult {ar_id}: {e}")
@@ -387,7 +376,7 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
             error_count += 1
             continue
 
-    print(f"✓ Migration complete: {migrated_count} migrated, {skipped_count} skipped, {error_count} errors")
+    print(f"    ✓ Process migration complete: {migrated_count} migrated, {skipped_count} skipped, {error_count} errors")
 
 
 class Migration(migrations.Migration):

archivebox/core/migrations/0029_migrate_archiveresult_to_uuid_pk.py

@@ -6,6 +6,9 @@ from uuid import UUID
 from archivebox.uuid_compat import uuid7
 
 
+PROGRESS_EVERY = 10000
+
+
 def migrate_archiveresult_id_to_uuid(apps, schema_editor):
     """
     Migrate ArchiveResult from integer PK to UUID PK (clean one-step migration).
@@ -28,7 +31,7 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
     # Check if table exists and has data
     cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_archiveresult'")
     if not cursor.fetchone():
-        print("ArchiveResult table does not exist, skipping migration")
+        print("    ✓ ArchiveResult table does not exist, skipping UUID PK migration")
         return
 
     cursor.execute("SELECT COUNT(*) FROM core_archiveresult")
@@ -38,16 +41,16 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
     # (fresh installs create table with uuid from 0025, but model expects no uuid after 0029)
 
     if row_count == 0:
-        print("[0029] Recreating ArchiveResult table schema (integer→UUID PK, removing uuid column)...")
+        print("    - Rebuilding empty ArchiveResult table with UUID primary keys...")
     else:
-        print(f"[0029] Migrating {row_count} ArchiveResult records from integer PK to UUID PK...")
+        print(f"    - Migrating {row_count} ArchiveResults from integer IDs to UUID primary keys...")
 
     # Step 0: Check if machine_process table exists, if not NULL out process_id values
     cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='machine_process'")
     machine_process_exists = cursor.fetchone() is not None
 
     if not machine_process_exists:
-        print("machine_process table does not exist yet, setting process_id to NULL")
+        print("      machine_process is unavailable; clearing process_id references...")
         cursor.execute("UPDATE core_archiveresult SET process_id = NULL WHERE process_id IS NOT NULL")
 
     # Step 1: Create new table with UUID as primary key (clean - no old_id or uuid columns)
@@ -149,10 +152,6 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
         # Build INSERT statement (only copy fields that exist in source)
         existing_fields = [f for f in fields_to_copy if f in values]
 
-        if i == 0:
-            print(f"[0029] Source columns: {col_names}")
-            print(f"[0029] Copying fields: {existing_fields}")
-
         placeholders = ", ".join(["?"] * (len(existing_fields) + 1))  # +1 for id
         field_list = "id, " + ", ".join(existing_fields)
 
@@ -164,13 +163,16 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
                 insert_values,
             )
             inserted_count += 1
+            if inserted_count % PROGRESS_EVERY == 0:
+                print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults...")
         except Exception as e:
             print(f"[0029] ERROR inserting record {old_id}: {e}")
             if i == 0:
                 print(f"[0029] First record values: {insert_values[:5]}...")
                 raise
 
-    print(f"[0029] Inserted {inserted_count}/{len(old_records)} records")
+    if old_records:
+        print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults")
 
     # Step 4: Replace old table with new table
     cursor.execute("DROP TABLE core_archiveresult")
@@ -185,7 +187,7 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
     cursor.execute("CREATE INDEX core_archiveresult_hook_name_idx ON core_archiveresult(hook_name)")
     cursor.execute("CREATE INDEX core_archiveresult_process_id_idx ON core_archiveresult(process_id)")
 
-    print(f"✓ Migrated {row_count} ArchiveResult records to UUID primary key")
+    print(f"    ✓ ArchiveResult UUID primary key migration complete ({row_count} records)")
 
 
 class Migration(migrations.Migration):

archivebox/core/migrations/0039_alter_archiveresult_process.py

@@ -0,0 +1,26 @@
+# Generated by Django 6.0.5 on 2026-05-27 20:11
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0038_snapshot_progress_idx"),
+        ("machine", "0015_process_progress_indexes"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="archiveresult",
+            name="process",
+            field=models.OneToOneField(
+                blank=True,
+                help_text="Process execution details for this archive result",
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="archiveresult",
+                to="machine.process",
+            ),
+        ),
+    ]

archivebox/core/migrations/0040_archiveresult_delete_at_snapshot_delete_at.py

@@ -0,0 +1,22 @@
+# Generated by Django 6.0.5 on 2026-05-27 20:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0039_alter_archiveresult_process"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="archiveresult",
+            name="delete_at",
+            field=models.DateTimeField(blank=True, db_index=True, default=None, null=True),
+        ),
+        migrations.AddField(
+            model_name="snapshot",
+            name="delete_at",
+            field=models.DateTimeField(blank=True, db_index=True, default=None, null=True),
+        ),
+    ]

archivebox/core/migrations/0041_snapshot_permissions.py

@@ -0,0 +1,23 @@
+# Generated by Django 6.0.5 on 2026-05-28 07:25
+
+import django.db.models.fields.json
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0040_archiveresult_delete_at_snapshot_delete_at"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="snapshot",
+            name="permissions",
+            field=models.GeneratedField(
+                db_index=True,
+                db_persist=True,
+                expression=django.db.models.fields.json.KeyTextTransform("PERMISSIONS", "config"),
+                output_field=models.CharField(max_length=16, null=True),
+            ),
+        ),
+    ]

archivebox/core/migrations/0042_snapshot_output_size.py

@@ -0,0 +1,43 @@
+# Generated by Django 6.0.5 on 2026-05-28 08:22
+
+from django.db import migrations, models
+from django.db.models import Sum
+
+
+def backfill_snapshot_output_size(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    ArchiveResult = apps.get_model("core", "ArchiveResult")
+    batch = []
+    rows = ArchiveResult.objects.values("snapshot_id").annotate(total_size=Sum("output_size")).order_by()
+    for row in rows.iterator(chunk_size=2000):
+        batch.append(Snapshot(id=row["snapshot_id"], output_size=row["total_size"] or 0))
+        if len(batch) >= 2000:
+            Snapshot.objects.bulk_update(batch, ["output_size"], batch_size=2000)
+            batch = []
+    if batch:
+        Snapshot.objects.bulk_update(batch, ["output_size"], batch_size=2000)
+
+
+def clear_snapshot_output_size(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    Snapshot.objects.update(output_size=0)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0041_snapshot_permissions"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="snapshot",
+            name="output_size",
+            field=models.BigIntegerField(
+                db_index=True,
+                default=0,
+                editable=False,
+                help_text="Total bytes of all ArchiveResult output files",
+            ),
+        ),
+        migrations.RunPython(backfill_snapshot_output_size, clear_snapshot_output_size),
+    ]