tweak release script

add harness tests
test fixes
2026-03-23 04:21:12 -07:00 · 2026-03-23 04:12:46 -07:00 · 2026-03-23 04:12:31 -07:00 · 2026-03-23 03:58:32 -07:00 · 2026-03-22 20:25:53 -07:00 · 2026-03-22 20:25:18 -07:00
681 changed files with 88150 additions and 33720 deletions
@@ -0,0 +1,5 @@
+{
+  "enabledPlugins": {
+    "pyright-lsp@claude-plugins-official": true
+  }
+}
@@ -0,0 +1,51 @@
+{
+  "permissions": {
+    "allow": [
+      "Read(**)",
+      "Glob(**)",
+      "Grep(**)",
+      "Bash(python -m archivebox:*)",
+      "Bash(ls:*)",
+      "Bash(xargs:*)",
+      "Bash(python -c:*)",
+      "Bash(printf:*)",
+      "Bash(pkill:*)",
+      "Bash(python3:*)",
+      "Bash(sqlite3:*)",
+      "WebFetch(domain:github.com)",
+      "Bash(uv add:*)",
+      "Bash(mkdir:*)",
+      "Bash(chmod:*)",
+      "Bash(python -m forum_dl:*)",
+      "Bash(archivebox manage migrate:*)",
+      "Bash(cat:*)",
+      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
+      "Bash(forum-dl:*)",
+      "Bash(pip uninstall:*)",
+      "Bash(python:*)",
+      "Bash(source .venv/bin/activate)",
+      "Bash(mv:*)",
+      "Bash(echo:*)",
+      "Bash(grep:*)",
+      "WebFetch(domain:python-statemachine.readthedocs.io)",
+      "Bash(./bin/run_plugin_tests.sh:*)",
+      "Bash(done)",
+      "Bash(coverage erase:*)",
+      "Bash(gh api:*)"
+    ]
+  },
+  "hooks": {
+    "PreToolUse": [
+      {
+        "matcher": "Bash",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
+            "statusMessage": "Checking working directory..."
+          }
+        ]
+      }
+    ]
+  }
+}
@@ -16,10 +16,12 @@ __pycache__/
 venv/
 .venv/
 .venv-old/
+.docker_venv/
 .docker-venv/
 node_modules/
 chrome/
 chromeprofile/
+chrome_profile/

 pdm.dev.lock
 pdm.lock
@@ -35,6 +37,7 @@ docker/
 website/
 typings/

+tmp/
 data/
 data*/
 output/
@@ -1,2 +1,2 @@
 github: ["ArchiveBox", "pirate"]
-custom: ["https://donate.archivebox.io", "https://paypal.me/NicholasSweeting"]
+custom: ["https://donate.archivebox.io", "https://swag.archivebox.io"]
@@ -27,10 +27,10 @@ body:
    id: steps_to_reproduce
    attributes:
      label: Steps to reproduce
-      description: Please include the exact steps you took to trigger the issue.
+      description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
      render: markdown
      placeholder: |
-        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:latest`
+        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox` in iTerm2
        2. Went to the https://127.0.0.1:8000/add/ page in Google Chrome
        3. Typed 'https://example.com/xyz' into the 'Add URL' input field
        4. Clicked the 'Add+' button
@@ -73,9 +73,9 @@ body:
        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
        
        Binary Dependencies:
-        √  python     3.11.9     venv_pip   ~/.venv/bin/python
-        √  django     5.1.2      venv_pip   ~/.venv/lib/python3.11/site-packages/django/__init__.py
-        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.11/site-packages/django/db/backends/sqlite3/base.py
+        √  python     3.14.0     venv_pip   ~/.venv/bin/python
+        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
+        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
        ...
    validations:
@@ -93,7 +93,7 @@ body:
        - apt
        - brew
        - nix
-        - Docker (or other container system like podman/LXC/Kubernetes or TrueNAS/Cloudron/YunoHost/etc.)
+        - Docker (or Podman/LXC/K8s/TrueNAS/Proxmox/etc)
        - Other

  - type: dropdown
@@ -116,15 +116,15 @@ body:
    id: filesystem
    attributes:
      label: What type of drive are you using to store your ArchiveBox data?
-      description: Are you using a [remote filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#supported-remote-filesystems) or FUSE mount for `data/` or `data/archive` (e.g. NFS/SMB/CIFS/etc. or FUSE/RClone/S3/B2/OneDrive/etc.)?
+      description: Are you using a [remote filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#supported-remote-filesystems) or FUSE mount for `data/` OR `data/archive`?
      options:
-        - label: "`data/` is on a local SSD or NVMe drive"
+        - label: "some of `data/` is on a local SSD or NVMe drive"
          required: false
-        - label: "`data/` is on a spinning hard drive or external USB drive"
+        - label: "some of `data/` is on a spinning hard drive or external USB drive"
          required: false
-        - label: "`data/` is on a network mount (e.g. NFS/SMB/CIFS/etc.)"
+        - label: "some of `data/` is on a network mount (e.g. NFS/SMB/Ceph/GlusterFS/etc.)"
          required: false
-        - label: "`data/` is on a FUSE mount (e.g. SSHFS/RClone/S3/B2/OneDrive, etc.)"
+        - label: "some of `data/` is on a FUSE mount (e.g. SSHFS/RClone/S3/B2/Google Drive/Dropbox/etc.)"
          required: false
  

@@ -24,7 +24,7 @@ body:
    attributes:
      label: "What is the problem that your feature request solves?"
      description: |
-        Describe the problem that your feature request solves, feel free to include any screenshots or examples.
+        Describe the problem or need that your feature request solves, feel free to include any screenshots or examples.
      placeholder: |
        e.g. I need to be able to archive spanish and french subtitle files from a particular movie site https://example.com/somevideos that's going down soon.
    validations:
@@ -35,7 +35,7 @@ body:
    attributes:
      label: "What is your proposed solution?"
      description: |
-        Describe the ideal specific solution you'd want, and whether it fits into any broader scope of changes.
+        Describe the ideal specific solution you'd want, *and whether it fits into any broader scope of changes*.
      placeholder: |
        e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
        The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
@@ -47,7 +47,7 @@ body:
    attributes:
      label: "What hacks or alternative solutions have you tried to solve the problem?"
      description: |
-        A clear and concise description of any alternative solutions, workarounds, or other software you've considered using to fix the problem.
+        A description of any alternative approaches, workarounds, or other solutions you've considered to fix the problem.
      placeholder: |
        e.g. I wait for archivebox to finish archiving the page, then I manually run `yt-dlp --subs <URL>` inside 
        the `data/archive/<timestamp>/` directory to download the subtitle files and add them to the snapshot folder.
@@ -57,10 +57,11 @@ body:
  - type: textarea
    id: version
    attributes:
-      label: What version of ArchiveBox are you currently using?
+      label: Share the entire output of the `archivebox version` command for the current version you are using.
      description: |
-        We need to know what version of ArchiveBox and what OS you're currently using in order to contextualize your feature request.  
-        Sometimes we've already fixed the issues in newer BETA versions, sometimes features already exist but may not be available in specific environments/versions.
+        DO NOT JUST ENTER "the latest version" OR YOUR ISSUE WILL BE CLOSED.  
+        We need to know what version of ArchiveBox and what feature flags you're currently running with in order to contextualize your feature request.  
+        Sometimes we've already fixed the issues in newer BETA versions, sometimes features already exist but may not be available in your specific environment.
          
        Run the `archivebox version` command inside your current collection dir and paste the *full output* here (*not just the version number*).  
        For Docker Compose run: `docker compose run archivebox version`  
@@ -74,9 +75,9 @@ body:
        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
        
        Binary Dependencies:
-        √  python     3.11.9     venv_pip   ~/.venv/bin/python
-        √  django     5.1.2      venv_pip   ~/.venv/lib/python3.11/site-packages/django/__init__.py
-        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.11/site-packages/django/db/backends/sqlite3/base.py
+        √  python     3.14.0     venv_pip   ~/.venv/bin/python
+        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
+        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
        ...
    validations:
@@ -93,7 +94,7 @@ body:
          required: false
        - label: "It would be nice to have eventually"
          required: false
-        - label: "I'm willing to [work on a PR](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) to develop this myself"
+        - label: "I'm willing to [start a PR](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) to develop this myself"
          required: false
        - label: "I have [donated money](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations) to go towards fixing this issue"
          required: false
@@ -1,8 +1,11 @@
 blank_issues_enabled: false
 contact_links:
-  - name: 💬 Chat with us on Zulip
+  - name: ❓ Ask a question or start a discussion
+    url: https://github.com/ArchiveBox/ArchiveBox/discussions
+    about: "Ask a question, get support, or start a design discussion (to report a problem please use '🐞 Bug report' instead)"
+  - name: 💬 Chat with the dev team & community on Zulip
    url: https://zulip.archivebox.io
    about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."
-  - name: 💁‍♂️ Hire us to provide archiving for your organization
+  - name: 💁‍♂️ Hire us for professional support with fast response times
    url: https://docs.monadical.com/s/archivebox-consulting-services
-    about: "We provide hosting, develoment, and support, including on-prem/cloud setup w/ SSO & storage, CAPTCHA/rate-limiting avoidance, etc."
+    about: "We provide hosting, development, and support, including on-prem/cloud w/ SSO & storage, CAPTCHA-solving, proxies, etc."
@@ -0,0 +1,49 @@
+name: Claude Code
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  issues:
+    types: [opened, assigned]
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  claude:
+    if: |
+      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
+      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+      issues: write
+      id-token: write
+      actions: read # Required for Claude to read CI results on PRs
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code
+        id: claude
+        uses: anthropics/claude-code-action@v1
+        with:
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+
+          # This is an optional setting that allows Claude to read CI results on PRs
+          additional_permissions: |
+            actions: read
+
+          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
+          # prompt: 'Update the pull request description to include a summary of changes.'
+
+          # Optional: Add claude_args to customize behavior and configuration
+          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
+          # or https://code.claude.com/docs/en/cli-reference for available options
+          claude_args: '--allowed-tools Bash(gh pr:*)'
@@ -2,63 +2,209 @@ name: Build Debian package

 on:
  workflow_dispatch:
+  workflow_call:
  push:
+    branches: ['**']
+    paths:
+      - 'pkg/debian/**'
+      - 'bin/build_deb.sh'
+      - 'bin/release_deb.sh'
+      - '.github/workflows/debian.yml'
+      - 'pyproject.toml'
+  # release trigger is handled by release.yml to avoid double-runs

-env:
-  DEB_BUILD_OPTIONS: nocheck
+permissions:
+  contents: write

 jobs:
  build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        arch: [amd64, arm64]

    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
        with:
-          submodules: true
          fetch-depth: 1

-      - name: Install packaging dependencies
+      - name: Get version
+        id: version
+        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
+
+      - name: Install nfpm
+        env:
+          NFPM_VERSION: "2.45.1"
        run: |
-          sudo apt-get update -qq
-          sudo apt-get install -y \
-            python3 python3-dev python3-pip python3-venv python3-all \
-            dh-python debhelper devscripts dput software-properties-common \
-            python3-distutils python3-setuptools python3-wheel python3-stdeb
+          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \
+            | sudo tar -xz -C /usr/local/bin nfpm
+          nfpm --version

-      # - name: Build Debian/Apt sdist_dsc
-      #   run: |
-      #     ./bin/build_pip.sh
+      - name: Build .deb package
+        run: |
+          export VERSION="${{ steps.version.outputs.version }}"
+          export ARCH="${{ matrix.arch }}"
+          ./bin/build_deb.sh

-      # - name: Check ArchiveBox version
-      #   run: |
-      #     # must create dir needed for snaps to run as non-root on github actions
-      #     sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
-      #     mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
-      #     archivebox --version
-      #     archivebox init --setup
+      - name: Verify .deb package contents
+        run: |
+          DEB_FILE="$(ls dist/archivebox*.deb | head -1)"
+          echo "=== Package info ==="
+          dpkg-deb --info "$DEB_FILE"
+          echo ""
+          echo "=== Package contents ==="
+          dpkg-deb --contents "$DEB_FILE"
+          echo ""
+          echo "=== Control fields ==="
+          dpkg-deb --field "$DEB_FILE"

-      # - name: Add some links to test
-      #   run: |
-      #     cd "${{ github.workspace }}/data"
-      #     archivebox add 'https://example.com'
-      #     archivebox status
+      - name: Upload .deb artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb
+          path: dist/*.deb

-      # - name: Commit built package
-      #   run: |
-      #     cd deb_dist/
-      #     git config --local user.email "action@github.com"
-      #     git config --local user.name "GitHub Action"
-      #     git commit -m "Debian package autobuild" -a
-      
-      # - name: Push build to Github
-      #   uses: ad-m/github-push-action@master
-      #   with:
-      #     github_token: ${{ secrets.GITHUB_TOKEN }}
-      #     repository: ArchiveBox/debian-archivebox
-      #     branch: ${{ github.ref }}
-      #     directory: deb_dist
+  test:
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            runner: ubuntu-24.04
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+    runs-on: ${{ matrix.runner }}

-      # - name: Push build to Launchpad PPA
-      #   run: |
-      #     debsign -k "$PGP_KEY_ID" "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
-      #     dput archivebox "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+
+      - name: Install build dependencies
+        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
+        with:
+          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
+          version: 1.0
+
+      - name: Build local wheel
+        run: |
+          uv sync --frozen --all-extras --no-install-project --no-install-workspace
+          uv build --wheel --out-dir /tmp/wheels/
+
+      - name: Download .deb artifact
+        uses: actions/download-artifact@v4
+        with:
+          pattern: archivebox-*-${{ matrix.arch }}.deb
+          merge-multiple: true
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget
+
+      - name: Pre-seed virtualenv with local wheel before dpkg install
+        run: |
+          # CI-only: pre-seed the venv with a local wheel so we test the
+          # unreleased code, not whatever is on PyPI. We explicitly use python3.13
+          # here (matching the system dep installed above) to ensure the venv is
+          # created with the correct Python version. On real installs, install.sh
+          # handles this by preferring python3.13 and failing if < 3.13.
+          # When postinstall.sh runs during dpkg -i, it finds the venv already
+          # populated and just upgrades deps.
+          sudo mkdir -p /opt/archivebox
+          sudo python3.13 -m venv /opt/archivebox/venv
+          sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools
+          sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl
+          echo "[√] Pre-seeded /opt/archivebox/venv with local wheel"
+
+      - name: Install .deb package
+        run: |
+          # dpkg install will run postinstall which creates the user,
+          # sets up systemd, and tries pip install (which finds it already installed)
+          sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y
+
+      - name: Verify archivebox is installed
+        run: |
+          which archivebox
+          archivebox version
+
+      - name: Verify wrapper script works
+        run: |
+          /usr/bin/archivebox version
+          /usr/bin/archivebox --help | head -5
+
+      - name: Test archivebox init as archivebox user
+        run: |
+          # The postinstall should have created the user
+          id archivebox
+          sudo mkdir -p /tmp/archivebox-test
+          sudo chown archivebox:archivebox /tmp/archivebox-test
+          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init'
+
+      - name: Test archivebox status
+        run: |
+          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status'
+
+      - name: Test archivebox add
+        run: |
+          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"'
+
+      - name: Verify systemd service file exists
+        run: |
+          test -f /usr/lib/systemd/system/archivebox.service
+          cat /usr/lib/systemd/system/archivebox.service
+
+  # Upload .deb to GitHub Release only after tests pass
+  release:
+    if: github.event_name == 'release' || github.event_name == 'workflow_call'
+    needs: [build, test]
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Determine release tag
+        id: tag
+        run: |
+          TAG="${{ github.event.release.tag_name }}"
+          if [ -z "$TAG" ]; then
+            TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')"
+            echo "[i] No release tag in event context, using version from pyproject.toml: $TAG"
+          fi
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Download all .deb artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: archivebox-*.deb
+          merge-multiple: true
+
+      - name: Upload .deb to GitHub Release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          TAG="${{ steps.tag.outputs.tag }}"
+          # Verify the release exists before uploading
+          if ! gh release view "$TAG" >/dev/null 2>&1; then
+            echo "[!] No GitHub Release found for tag $TAG."
+            if [ -n "${{ github.event.release.tag_name }}" ]; then
+              echo "[X] This was triggered by a release event — the release should exist. Failing."
+              exit 1
+            fi
+            echo "[i] Skipping upload (workflow_dispatch without a release)."
+            echo "    Create a release first or trigger via the release event."
+            exit 0
+          fi
+          gh release upload "$TAG" *.deb --clobber
@@ -2,6 +2,7 @@ name: Build Docker image

 on:
  workflow_dispatch:
+  workflow_call:
  push:
    branches:
      - '**'
@@ -18,9 +19,9 @@ jobs:
    steps:
      - name: Checkout
        uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
+        # with:
+        #   submodules: true
+        #   fetch-depth: 1

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
@@ -40,26 +41,34 @@ jobs:
        run: echo ${{ steps.buildx.outputs.platforms }}

      - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@v4
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-

-      - name: Docker Login
+      - name: Login to Docker Hub
        uses: docker/login-action@v3
        if: github.event_name != 'pull_request'
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

-      - name: Collect Docker tags
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Collect Full Release Docker tags
        # https://github.com/docker/metadata-action
        id: docker_meta
        uses: docker/metadata-action@v5
+        if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
        with:
-          images: archivebox/archivebox,nikisweeting/archivebox
+          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
          tags: |
              # :stable
              type=ref,event=branch
@@ -72,6 +81,19 @@ jobs:
              # :latest
              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}

+      - name: Collect Non-Release Docker tags
+        # https://github.com/docker/metadata-action
+        id: docker_meta_non_release
+        uses: docker/metadata-action@v5
+        if: github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call'
+        with:
+          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
+          tags: |
+              # :stable
+              type=ref,event=branch
+              # :sha-463ea54
+              type=sha
+
      - name: Build and push
        id: docker_build
        uses: docker/build-push-action@v5
@@ -80,8 +102,8 @@ jobs:
          file: ./Dockerfile
          builder: ${{ steps.buildx.outputs.name }}
          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
+          tags: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.tags || steps.docker_meta_non_release.outputs.tags }}
+          labels: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.labels || steps.docker_meta_non_release.outputs.labels }}
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,dest=/tmp/.buildx-cache-new
          platforms: linux/amd64,linux/arm64
@@ -0,0 +1,59 @@
+name: Duplicate Issue Detection
+
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  check-duplicates:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Install opencode
+        run: curl -fsSL https://opencode.ai/install | bash
+
+      - name: Check for duplicate issues
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OPENCODE_PERMISSION: |
+            {
+              "bash": {
+                "gh issue*": "allow",
+                "gh pr*": "allow",
+                "*": "deny"
+              },
+              "webfetch": "allow"
+            }
+        run: |
+          opencode run -m anthropic/claude-haiku-4-5 "A new issue has been created:
+
+          Issue number:
+          ${{ github.event.issue.number }}
+
+          Lookup this issue and search through existing issues and PRs (excluding #${{ github.event.issue.number }}) in this repository to find any potential duplicates of this new issue.
+          Consider:
+          1. Similar titles or descriptions
+          2. Same error messages or symptoms
+          3. Related functionality or components
+          4. Similar feature requests
+
+          If you find any potential duplicates, please comment on the new issue with:
+          - A brief explanation of why it might be a duplicate
+          - Links to the potentially duplicate issues or PRs
+          - A suggestion to check those issues first
+
+          Use this format for the comment:
+          'This issue might be a duplicate of existing issues. Please check:
+          - #[issue_number]: [brief description of similarity]
+
+          Feel free to ignore if none of these address your specific case.'
+
+          If no clear duplicates are found, do not comment."
@@ -40,14 +40,14 @@ jobs:
          rm -f ./website/README.md
          cp ./README.md ./website/README.md
      - name: Setup Pages
-        uses: actions/configure-pages@v3
+        uses: actions/configure-pages@v5
      - name: Build with Jekyll
        uses: actions/jekyll-build-pages@v1
        with:
          source: ./website
          destination: ./_site
      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v2
+        uses: actions/upload-pages-artifact@v3

      # - name: Checkout
      #   uses: actions/checkout@v4
@@ -1,51 +1,229 @@
-name: Build Homebrew package
+name: Build Homebrew formula

 on:
  workflow_dispatch:
+  workflow_call:
  push:
+    branches: ['**']
+    paths:
+      - 'brew_dist/**'
+      - 'bin/build_brew.sh'
+      - 'bin/release_brew.sh'
+      - '.github/workflows/homebrew.yml'
+      - 'pyproject.toml'
+  # release trigger is handled by release.yml to avoid double-runs

+permissions:
+  contents: read

 jobs:
-  build:
+  build-and-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-24.04]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Get version
+        id: version
+        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
+
+      - name: Validate formula template syntax
+        run: ruby -c brew_dist/archivebox.rb
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+
+      - name: Install build dependencies (Linux)
+        if: runner.os == 'Linux'
+        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
+        with:
+          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
+          version: 1.0
+
+      - name: Build local sdist
+        run: |
+          uv sync --frozen --all-extras --no-install-project --no-install-workspace
+          uv build --sdist --out-dir /tmp/sdist/
+
+      - name: Generate formula from local sdist
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)"
+          SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')"
+
+          # Install archivebox + poet into a temp venv to generate resource stanzas
+          python3 -m venv /tmp/poet-venv
+          source /tmp/poet-venv/bin/activate
+          pip install --quiet "$SDIST_PATH" homebrew-pypi-poet
+          echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
+          RESOURCES="$(poet archivebox)"
+          deactivate
+
+          # For CI: use file:// URL pointing to local sdist
+          # For release: this gets overridden with the PyPI URL
+          SDIST_URL="file://${SDIST_PATH}"
+
+          cat > /tmp/archivebox.rb << RUBY
+class Archivebox < Formula
+  include Language::Python::Virtualenv
+
+  desc "Self-hosted internet archiving solution"
+  homepage "https://github.com/ArchiveBox/ArchiveBox"
+  url "${SDIST_URL}"
+  sha256 "${SDIST_SHA256}"
+  license "MIT"
+
+  depends_on "python@3.13"
+
+${RESOURCES}
+
+  def install
+    virtualenv_install_with_resources
+  end
+
+  def post_install
+    data_dir = var/"archivebox"
+    data_dir.mkpath
+    ENV["DATA_DIR"] = data_dir.to_s
+    system bin/"archivebox", "init"
+  end
+
+  def caveats
+    <<~EOS
+      ArchiveBox data is stored in:
+        #{var}/archivebox
+
+      To start archiving, run:
+        cd #{var}/archivebox && archivebox add 'https://example.com'
+
+      To start the web UI:
+        cd #{var}/archivebox && archivebox server 0.0.0.0:8000
+    EOS
+  end
+
+  test do
+    assert_match version.to_s, shell_output("#{bin}/archivebox version")
+  end
+end
+RUBY
+
+          echo "[√] Generated formula:"
+          ruby -c /tmp/archivebox.rb
+          cat /tmp/archivebox.rb
+
+      - name: Install Homebrew (Linux only)
+        if: runner.os == 'Linux'
+        run: |
+          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
+          echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV"
+          echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV"
+          echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV"
+          echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH"
+          echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH"
+
+      - name: Install brew dependencies
+        run: brew install python@3.13
+
+      - name: Install archivebox via brew from local formula
+        run: |
+          brew install --build-from-source --verbose /tmp/archivebox.rb
+
+      - name: Verify archivebox version
+        run: archivebox version
+
+      - name: Test archivebox init
+        run: |
+          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
+          archivebox init
+
+      - name: Test archivebox status
+        run: |
+          cd /tmp/archivebox-test
+          archivebox status
+
+  # On release only: generate the real formula with PyPI URL and push to tap
+  release:
+    if: github.event_name == 'release' || github.event_name == 'workflow_call'
+    needs: build-and-test
    runs-on: macos-latest

    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
        with:
-          submodules: true
          fetch-depth: 1

-      # TODO: modify archivebox.rb to update src url, hashes, and dependencies
+      - name: Get version
+        id: version
+        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"

-      - name: Build Homebrew Bottle
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Wait for PyPI package availability
        run: |
-          pip3 install --upgrade pip setuptools wheel
-          cd brew_dist/
-          brew install --build-bottle ./archivebox.rb
-          # brew bottle archivebox
+          VERSION="${{ steps.version.outputs.version }}"
+          echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..."
+          for i in $(seq 1 30); do
+            if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then
+              echo "[√] archivebox==${VERSION} is available on PyPI"
+              break
+            fi
+            if [ "$i" -eq 30 ]; then
+              echo "[!] Timed out waiting for PyPI. Trying to install anyway..."
+              break
+            fi
+            echo "    Attempt $i/30 - not yet available, waiting 30s..."
+            sleep 30
+          done
+
+      - name: Generate release formula via build_brew.sh
+        run: ./bin/build_brew.sh
+
+      - name: Test formula install
+        run: |
+          brew install --build-from-source brew_dist/archivebox.rb
          archivebox version

-      - name: Add some links to test
+      - name: Upload formula artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: archivebox.rb
+          path: brew_dist/archivebox.rb
+
+      - name: Push to homebrew-archivebox tap
+        env:
+          GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
        run: |
-          mkdir data && cd data
-          archivebox init --setup
-          archivebox add 'https://example.com'
-          archivebox version
-          archivebox status
+          VERSION="${{ steps.version.outputs.version }}"
+          git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap

-      # - name: Commit built package
-      #   run: |
-      #     cd brew_dist/
-      #     git config --local user.email "action@github.com"
-      #     git config --local user.name "GitHub Action"
-      #     git commit -m "Homebrew package autobuild" -a
-      
-      # - name: Push build to Github
-      #   uses: ad-m/github-push-action@master
-      #   with:
-      #     github_token: ${{ secrets.GITHUB_TOKEN }}
-      #     repository: ArchiveBox/homebrew-archivebox
-      #     branch: ${{ github.ref }}
-      #     directory: brew_dist
+          cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \
+            cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb

-      # TODO: push bottle homebrew core PR with latest changes
+          cd /tmp/tap
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add -A
+          if git diff --cached --quiet; then
+            echo "[i] No changes to formula, skipping push."
+          else
+            git commit -m "Update archivebox to v${VERSION}"
+            git push origin HEAD
+            echo "[√] Formula pushed to homebrew-archivebox tap"
+          fi
@@ -36,14 +36,14 @@ jobs:
          rm ./website/README.md
          cp ./README.md ./website/README.md
      - name: Setup Pages
-        uses: actions/configure-pages@v3
+        uses: actions/configure-pages@v5
      - name: Build with Jekyll
        uses: actions/jekyll-build-pages@v1
        with:
          source: ./website
          destination: ./_site
      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v2
+        uses: actions/upload-pages-artifact@v3

  # Deployment job
  deploy:
@@ -55,4 +55,4 @@ jobs:
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
-        uses: actions/deploy-pages@v2
+        uses: actions/deploy-pages@v4
@@ -4,32 +4,28 @@ on:
  workflow_dispatch:
  push:

-env:
-  MAX_LINE_LENGTH: 110
-
 jobs:
  lint:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 1

      - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
        with:
-          python-version: 3.9
+          python-version: "3.13"
          architecture: x64

-      - name: Install flake8
-        run: |
-          pip install flake8
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true

-      - name: Lint with flake8
-        run: |
-          cd archivebox
-          # one pass for show-stopper syntax errors or undefined names
-          flake8 . --count --show-source --statistics
-          # one pass for small stylistic things
-          flake8 . --count --max-line-length="$MAX_LINE_LENGTH" --statistics
+      - name: Install dependencies with uv
+        run: uv sync --all-extras --all-groups --no-sources --no-cache
+
+      - name: Run prek
+        run: uv run prek run --all-files
@@ -2,77 +2,64 @@ name: Build Pip package

 on:
  workflow_dispatch:
+  workflow_call:
  push:
+    branches:
+      - '**'
+    tags:
+      - 'v*'

+env:
+  PYTHON_VERSION: "3.13"

 jobs:
  build:
-    runs-on: ubuntu-22.04
-
    permissions:
      id-token: write

+    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
        with:
-          submodules: true
-          fetch-depth: 1
+          enable-cache: true

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
-          python-version: '3.11'
+          python-version: ${{ env.PYTHON_VERSION }}
          architecture: x64

-      - uses: awalsh128/cache-apt-pkgs-action@latest
+      - name: APT install archivebox dev + run dependencies
+        uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

-      - uses: pdm-project/setup-pdm@v3
+      - name: UV install archivebox dev + run sub-dependencies
+        run: uv sync --frozen --all-extras --no-install-project --no-install-workspace
+
+      - name: UV build archivebox and archivebox/pkgs/* packages
+        run: |
+          uv build --all
+
+      - name: Publish new package wheels and sdists to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # ignore when publish to PyPI fails due to duplicate tag
+        continue-on-error: true
        with:
-          python-version: '3.11'
-          cache: true
+          password: ${{ secrets.PYPI_PAT_SECRET }}

-      - name: Install dependencies
-        run: pdm install --fail-fast --no-lock --dev --group=':all' --no-self
+      - name: UV install archivebox and archivebox/pkgs/* locally for tests
+        run: uv sync --frozen --all-extras

-      - name: Build package
+      - name: UV run archivebox init + archivebox version
        run: |
-          pdm build
-
-      - name: Install from build
-        run: pip install ./dist/archivebox-*.whl
-
-      - name: Add some links to test
-        run: |
-          mkdir data && cd data
-          archivebox init
-          archivebox add 'https://example.com'
-          archivebox version
-          archivebox status
-
-      #- name: Publish package distributions to PyPI
-      #  run: pdm publish --no-build
-
-      # - name: Push build to PyPI
-      #   run: |
-      #     cd pip_dist/
-      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
-      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}
-
-      # - name: Commit built package
-      #   run: |
-      #     cd pip_dist/
-      #     git config --local user.email "action@github.com"
-      #     git config --local user.name "GitHub Action"
-      #     git commit -m "Pip package autobuild" -a
-      
-      # - name: Push build to Github
-      #   uses: ad-m/github-push-action@master
-      #   with:
-      #     github_token: ${{ secrets.GITHUB_TOKEN }}
-      #     repository: ArchiveBox/pip-archivebox
-      #     branch: ${{ github.ref }}
-      #     directory: pip_dist
-
+          mkdir -p data && cd data
+          uv run archivebox init \
+          && uv run archivebox version
+          # && uv run archivebox add 'https://example.com' \
+          # && uv run archivebox status \
+          # || (echo "UV Failed to run archivebox!" && exit 1)
@@ -0,0 +1,45 @@
+name: Release State
+
+on:
+  push:
+    branches:
+      - '**'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  id-token: write
+
+jobs:
+  release-state:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+          ref: ${{ github.ref_name }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Configure git identity
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Run release script
+        env:
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+          GH_TOKEN: ${{ github.token }}
+          PYPI_PAT_SECRET: ${{ secrets.PYPI_PAT_SECRET }}
+        run: ./bin/release.sh
@@ -0,0 +1,42 @@
+name: Release
+
+# Orchestrates the full release pipeline:
+#   1. Build and publish pip package to PyPI
+#   2. Build .deb packages and Homebrew formula (in parallel, after pip)
+#   3. Build Docker images (in parallel with deb/brew)
+#
+# Individual workflows also run on push for CI (see their own triggers).
+# This workflow ensures the correct ordering during a release.
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: write
+  packages: write
+  id-token: write
+
+jobs:
+  pip:
+    name: Publish to PyPI
+    uses: ./.github/workflows/pip.yml
+    secrets: inherit
+
+  debian:
+    name: Build .deb packages
+    needs: pip
+    uses: ./.github/workflows/debian.yml
+    secrets: inherit
+
+  homebrew:
+    name: Update Homebrew formula
+    needs: pip
+    uses: ./.github/workflows/homebrew.yml
+    secrets: inherit
+
+  docker:
+    name: Build Docker images
+    needs: pip
+    uses: ./.github/workflows/docker.yml
+    secrets: inherit
@@ -0,0 +1,158 @@
+name: Parallel Tests
+
+on:
+  pull_request:
+    branches: [dev, main, master]
+  push:
+    branches: [dev]
+
+env:
+  PYTHONIOENCODING: utf-8
+  PYTHONLEGACYWINDOWSSTDIO: utf-8
+  USE_COLOR: False
+
+jobs:
+  discover-tests:
+    name: Discover test files
+    runs-on: ubuntu-22.04
+    outputs:
+      test-files: ${{ steps.set-matrix.outputs.test-files }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Discover test files
+        id: set-matrix
+        run: |
+          # Find all main test files
+          all_tests=$(find archivebox/tests -maxdepth 1 -name "test_*.py" -type f | sort)
+
+          # Create JSON array with test file info
+          json_array="["
+          first=true
+          for test_file in $all_tests; do
+            if [ "$first" = true ]; then
+              first=false
+            else
+              json_array+=","
+            fi
+
+            # Extract a display name for the test
+            name="main/$(basename $test_file .py | sed 's/^test_//')"
+
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
+          done
+          json_array+="]"
+
+          echo "test-files=$json_array" >> $GITHUB_OUTPUT
+          echo "Found $(echo $all_tests | wc -w) test files"
+          echo "$json_array" | jq '.'
+
+  run-tests:
+    name: ${{ matrix.test.name }}
+    runs-on: ubuntu-22.04
+    needs: discover-tests
+
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
+        python: ["3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Cache uv
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml', 'uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python }}-uv-
+
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.1
+
+      - name: Install dependencies with uv
+        run: |
+          uv sync --dev --all-extras --no-sources
+
+      - name: Run test - ${{ matrix.test.name }}
+        run: |
+          mkdir -p tests/out
+          uv run pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
+
+  plugin-tests:
+    name: Plugin tests
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Set up Python 3.13
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.13"
+          architecture: x64
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Cache uv
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml', 'uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-3.13-uv-
+
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.1
+
+      - name: Install dependencies with uv
+        run: |
+          uv sync --dev --all-extras --no-sources
+
+      - name: Run plugin tests
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
+        run: |
+          uv run bash ./bin/test_plugins.sh --no-coverage
@@ -15,7 +15,7 @@ jobs:
      matrix:
        os: [ubuntu-22.04]
        # os: [ubuntu-22.04, macos-latest, windows-latest]
-        python: [3.11]
+        python: ["3.13"]

    steps:
      - uses: actions/checkout@v4
@@ -30,63 +30,33 @@ jobs:
          python-version: ${{ matrix.python }}
          architecture: x64

+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
      - name: Set up Node JS
        uses: actions/setup-node@v4
        with:
-          node-version: 20.10.0
-
-      - name: Setup PDM
-        uses: pdm-project/setup-pdm@v3
-        with:
-          python-version: '3.11'
-          cache: true
+          node-version: 22

      ### Install Python & JS Dependencies
-      - name: Get pip cache dir
-        id: pip-cache
-        run: |
-          echo "::set-output name=dir::$(pip cache dir)"
-
-      - name: Cache pip
+      - name: Cache uv
        uses: actions/cache@v3
-        id: cache-pip
        with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-${{ matrix.python }}-venv-${{ hashFiles('setup.py') }}
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml', 'uv.lock') }}
          restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-venv-
+            ${{ runner.os }}-${{ matrix.python }}-uv-

      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

-      - name: Install pip dependencies
+      - name: Install dependencies with uv
        run: |
-          python -m pip install --upgrade pip setuptools wheel pytest bottle build
-          python -m pip install -r requirements.txt
-          python -m pip install -e .[sonic,ldap]
-
-      - name: Get npm cache dir
-        id: npm-cache
-        run: |
-          echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
-
-      - name: Cache npm
-        uses: actions/cache@v3
-        id: cache-npm
-        with:
-          path: ${{ steps.npm-cache.outputs.dir }}
-          key: ${{ runner.os }}-node_modules-${{ hashFiles('package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node_modules
-
-      - name: Install npm requirements
-        run: |
-          npm install
-          echo "SINGLEFILE_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/single-file" >> $GITHUB_ENV
-          echo "READABILITY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/readability-extractor" >> $GITHUB_ENV
-          echo "MERCURY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/mercury-parser" >> $GITHUB_ENV
+          uv sync --dev --all-extras --no-sources

      ### Run the tests
      - name: Directory listing for debugging
@@ -96,13 +66,24 @@ jobs:

      - name: Archivebox version
        run: |
-          archivebox version
+          mkdir -p tests/out/data
+          DATA_DIR="$PWD/tests/out/data" uv run archivebox version

      - name: Test built package with pytest
        # TODO: remove this exception for windows once we get tests passing on that platform
        if: ${{ !contains(matrix.os, 'windows') }}
        run: |
-          python -m pytest -s --basetemp=tests/out --ignore=archivebox/pkgs
+          mkdir -p tests/out
+          uv run pytest -s archivebox/tests --basetemp=tests/out --ignore=archivebox/pkgs
+
+      - name: Run plugin tests
+        if: ${{ !contains(matrix.os, 'windows') }}
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
+        run: |
+          uv run bash ./bin/test_plugins.sh --no-coverage

  docker_tests:
    runs-on: ubuntu-latest
@@ -6,6 +6,13 @@ __pycache__/
 .eggs/
 tests/out/

+# Coverage
+.coverage
+.coverage.*
+coverage.json
+coverage/
+htmlcov/
+
 # Python and Node dependencies
 venv/
 .venv/
@@ -18,6 +25,7 @@ pdm.dev.lock
 requirements-dev.txt

 # Packaging artifacts
+requirements.txt
 .pdm-python
 .pdm-build
 archivebox.egg-info
@@ -30,12 +38,20 @@ lib/
 tmp/
 data/
 data*/
+archivebox/tests/data/
+archive/
 output/
+logs/
 index.sqlite3
 queue.sqlite3
 *.sqlite*
 data.*
 .archivebox_id
+ArchiveBox.conf
+*.stdout
+*.stderr
+*.log
+.tmp/

 # vim
 *.sw?
@@ -0,0 +1,67 @@
+default_language_version:
+  python: python3.13
+
+repos:
+  - repo: https://github.com/asottile/yesqa
+    rev: v1.5.0
+    hooks:
+      - id: yesqa
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.1
+    hooks:
+      - id: codespell
+        additional_dependencies:
+          - tomli
+
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.20.0
+    hooks:
+      - id: pyupgrade
+        args: [--py313-plus]
+
+  - repo: https://github.com/asottile/add-trailing-comma
+    rev: v3.1.0
+    hooks:
+      - id: add-trailing-comma
+
+  - repo: local
+    hooks:
+      - id: ruff-format
+        name: ruff-format
+        entry: uv run --active ruff format
+        language: system
+        types_or: [python, pyi]
+      - id: ruff-check
+        name: ruff-check
+        entry: uv run --active ruff check --fix
+        language: system
+        types_or: [python, pyi]
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-ast
+      - id: check-toml
+      - id: check-yaml
+        exclude: ^\.github/workflows/homebrew\.yml$
+      - id: check-json
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: destroyed-symlinks
+      - id: check-case-conflict
+      - id: check-illegal-windows-names
+      - id: check-shebang-scripts-are-executable
+        exclude: ^(archivebox/.*\.py|archivebox/tests/.*\.py|archivebox/personas/export_browser_state\.js)$
+      - id: mixed-line-ending
+      - id: fix-byte-order-marker
+      - id: end-of-file-fixer
+      - id: detect-private-key
+      - id: debug-statements
+      - id: forbid-submodules
+        exclude: ^docs$
+      - id: check-added-large-files
+        args: ["--maxkb=600"]
+      - id: name-tests-test
+        args: ["--pytest-test-first"]
+        exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$)
@@ -0,0 +1,497 @@
+# Claude Code Development Guide for ArchiveBox
+
+## Quick Start
+
+```bash
+# Set up dev environment (always use uv, never pip directly)
+uv sync --dev --all-extras
+
+# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
+```
+
+## Development Environment Setup
+
+### Prerequisites
+- Python 3.11+ (3.13 recommended)
+- uv package manager
+- A non-root user for running tests (e.g., `testuser`)
+
+### Install Dependencies
+```bash
+uv sync --dev --all-extras  # Always use uv, never pip directly
+```
+
+### Activate Virtual Environment
+```bash
+source .venv/bin/activate
+```
+
+### Common Gotchas
+
+#### File Permissions
+New files created by root need permissions fixed for testuser:
+```bash
+chmod 644 archivebox/tests/test_*.py
+```
+
+#### DATA_DIR Environment Variable
+ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
+
+## Code Style Guidelines
+
+### Naming Conventions for Grep-ability
+Use consistent naming for everything to enable easy grep-ability and logical grouping:
+
+**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
+
+**Examples**:
+```python
+# Filesystem migration methods - all start with fs_
+def fs_migration_needed() -> bool: ...
+def fs_migrate() -> None: ...
+def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
+def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
+def _fs_next_version(current: str) -> str: ...
+
+# Logging methods - ALL must start with log_ or _log
+def log_migration_start(snapshot_id: str) -> None: ...
+def _log_error(message: str) -> None: ...
+def log_validation_result(ok: bool, msg: str) -> None: ...
+```
+
+**Rules**:
+- Group related functions with common prefixes
+- Use `_` prefix for internal/private helpers within the same family
+- ALL logging-related methods MUST start with `log_` or `_log`
+- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
+- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
+
+### Minimize Unique Names and Data Structures
+**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
+
+**Example - GOOD**:
+```python
+# Binary has overrides field
+binary = Binary(overrides={'TIMEOUT': '60s'})
+
+# Binary reuses the same field name and structure
+class Binary(models.Model):
+    overrides = models.JSONField(default=dict)  # Same name, same structure
+```
+
+**Example - BAD**:
+```python
+# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
+class Binary(models.Model):
+    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
+```
+
+**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
+
+## Testing
+
+### CRITICAL: Never Run as Root
+ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
+
+```bash
+# Run all migration tests
+sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
+
+# Run specific test file
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
+
+# Run single test
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
+```
+
+### Test File Structure
+```
+archivebox/tests/
+├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
+├── test_migrations_fresh.py      # Fresh install tests
+├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
+├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
+└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
+```
+
+### Test Writing Standards
+
+#### NO MOCKS - Real Tests Only
+Tests must exercise real code paths:
+- Create real SQLite databases with version-specific schemas
+- Seed with realistic test data
+- Run actual `python -m archivebox` commands via subprocess
+- Query SQLite directly to verify results
+
+**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
+
+#### NO SKIPS
+Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
+
+#### Strict Assertions
+- `init` command must return exit code 0 (not `[0, 1]`)
+- Verify ALL data is preserved, not just "at least one"
+- Use exact counts (`==`) not loose bounds (`>=`)
+
+### Example Test Pattern
+```python
+def test_migration_preserves_snapshots(self):
+    """Migration should preserve all snapshots."""
+    result = run_archivebox(self.work_dir, ['init'], timeout=45)
+    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
+
+    ok, msg = verify_snapshot_count(self.db_path, expected_count)
+    self.assertTrue(ok, msg)
+```
+
+### Testing Gotchas
+
+#### Extractors Disabled for Speed
+Tests disable all extractors via environment variables for faster execution:
+```python
+env['SAVE_TITLE'] = 'False'
+env['SAVE_FAVICON'] = 'False'
+# ... etc
+```
+
+#### Timeout Settings
+Use appropriate timeouts for migration tests (45s for init, 60s default).
+
+### Plugin Testing & Code Coverage
+
+**Target: 80-90% coverage** for critical plugins (screenshot, chrome, singlefile, dom)
+
+```bash
+# Run plugin tests with coverage (both Python + JavaScript)
+bash bin/test_plugins.sh screenshot
+
+# View coverage reports
+bash bin/test_plugins.sh --coverage-report
+# Or individual reports:
+coverage report --show-missing --include='archivebox/plugins/*' --omit='*/tests/*'
+```
+
+#### Plugin Test Structure
+
+Tests are **completely isolated** from ArchiveBox - they replicate production directory structure in temp dirs:
+
+```python
+# Correct production paths:
+# Crawl:    DATA_DIR/users/{username}/crawls/YYYYMMDD/example.com/{crawl-id}/{plugin}/
+# Snapshot: DATA_DIR/users/{username}/snapshots/YYYYMMDD/example.com/{snapshot-uuid}/{plugin}/
+
+with tempfile.TemporaryDirectory() as tmpdir:
+    data_dir = Path(tmpdir)
+
+    # Crawl-level plugin (e.g., chrome launcher)
+    crawl_dir = data_dir / 'users' / 'testuser' / 'crawls' / '20240101' / 'example.com' / 'crawl-123'
+    chrome_dir = crawl_dir / 'chrome'
+    chrome_dir.mkdir(parents=True)
+
+    # Snapshot-level plugin (e.g., screenshot)
+    snapshot_dir = data_dir / 'users' / 'testuser' / 'snapshots' / '20240101' / 'example.com' / 'snap-456'
+    screenshot_dir = snapshot_dir / 'screenshot'
+    screenshot_dir.mkdir(parents=True)
+
+    # Run hook in its output directory
+    result = subprocess.run(
+        ['node', str(SCREENSHOT_HOOK), '--url=https://example.com'],
+        cwd=str(screenshot_dir),
+        env={**get_test_env(), 'EXTRA_CONTEXT': '{"snapshot_id":"snap-456"}'},
+        capture_output=True,
+        timeout=120
+    )
+```
+
+#### Coverage Improvement Loop
+
+To improve from ~20% to 80%+:
+
+1. **Run tests**: `bash bin/test_plugins.sh screenshot` → Shows: `19.1% (13/68 ranges)`
+2. **Identify gaps**: Check hook file for untested paths (session connection vs fallback, config branches, error cases)
+3. **Add tests**: Test both execution paths (connect to session + launch own browser), skip conditions, error cases, config variations
+4. **Verify**: Re-run tests → Should show: `85%+ (58+/68 ranges)`
+
+**Critical**: JavaScript hooks have TWO paths that both must be tested (connect to session ~50% + launch browser ~30% + shared ~20%). Testing only one path = max 50% coverage possible!
+
+## Database Migrations
+
+### Generate and Apply Migrations
+```bash
+# Generate migrations (run from archivebox subdirectory)
+cd archivebox
+./manage.py makemigrations
+
+# Apply migrations to test database
+cd data/
+archivebox init
+```
+
+### Schema Versions
+- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
+- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
+- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
+- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
+
+### Testing a Migration Path
+1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
+2. Seed with realistic test data using `seed_0_X_data()`
+3. Run `archivebox init` to trigger migrations
+4. Verify data preservation with `verify_*` functions
+5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
+
+### Squashed Migrations
+When testing 0.8.x (dev branch), you must record ALL replaced migrations:
+```python
+# The squashed migration replaces these - all must be recorded
+('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
+('core', '0024_auto_20240513_1143'),
+# ... all 52 migrations from 0023-0074 ...
+('core', '0023_new_schema'),  # Also record the squashed migration itself
+```
+
+### Migration Strategy
+- Squashed migrations for clean installs
+- Individual migrations recorded for upgrades from dev branch
+- `replaces` attribute in squashed migrations lists what they replace
+
+### Migration Gotchas
+
+#### Circular FK References in Schemas
+SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
+
+## Plugin System Architecture
+
+### Plugin Dependency Rules
+
+Like other plugins, chrome plugins **ARE NOT ALLOWED TO DEPEND ON ARCHIVEBOX OR DJANGO**.
+However, they are allowed to depend on two shared files ONLY:
+- `archivebox/plugins/chrome/chrome_utils.js` ← source of truth API for all basic chrome ops
+- `archivebox/plugins/chrome/tests/chrome_test_utils.py` ← use for your tests, do not implement launching/killing/pid files/cdp/etc. in python, just extend this file as needed.
+
+### Chrome-Dependent Plugins
+
+Many plugins depend on Chrome/Chromium via CDP (Chrome DevTools Protocol). When checking for script name references or debugging Chrome-related issues, check these plugins:
+
+**Main puppeteer-based chrome installer + launcher plugin**:
+- `chrome` - Core Chrome integration (CDP, launch, navigation)
+
+**Metadata extraction using chrome/chrome_utils.js / CDP**:
+- `dns` - DNS resolution info
+- `ssl` - SSL certificate info
+- `headers` - HTTP response headers
+- `redirects` - Capture redirect chains
+- `staticfile` - Direct file downloads (e.g. if the url itself is a .png, .exe, .zip, etc.)
+- `responses` - Capture network responses
+- `consolelog` - Capture console.log output
+- `title` - Extract page title
+- `accessibility` - Extract accessibility tree
+- `seo` - Extract SEO metadata
+
+**Extensions installed using chrome/chrome_utils.js / controlled using CDP**:
+- `ublock` - uBlock Origin ad blocking
+- `istilldontcareaboutcookies` - Cookie banner dismissal
+- `twocaptcha` - 2captcha CAPTCHA solver integration
+
+**Page-alteration plugins to prepare the content for archiving**:
+- `modalcloser` - Modal dialog dismissal
+- `infiniscroll` - Infinite scroll handler
+
+**Main Extractor Outputs**:
+- `dom` - DOM snapshot extraction
+- `pdf` - Generate PDF snapshots
+- `screenshot` - Generate screenshots
+- `singlefile` - SingleFile archival, can be single-file-cli that launches chrome, or singlefile extension running inside chrome
+
+**Crawl URL parsers** (post-process dom.html, singlefile.html, staticfile, responses, headers, etc. for URLs to re-emit as new queued Snapshots during recursive crawling):
+- `parse_dom_outlinks` - Extract outlinks from DOM (special, uses CDP to directly query browser)
+- `parse_html_urls` - Parse URLs from HTML (doesn't use chrome directly, just reads dom.html)
+- `parse_jsonl_urls` - Parse URLs from JSONL (doesn't use chrome directly, just reads dom.html)
+- `parse_netscape_urls` - Parse Netscape bookmark format (doesn't use chrome directly, just reads dom.html)
+
+### Finding Chrome-Dependent Plugins
+
+```bash
+# Find all files containing "chrom" (case-insensitive)
+grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d: -f1 | sort -u
+
+# Or get just the plugin names
+grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d/ -f3 | sort -u
+```
+
+**Note**: This list may not be complete. Always run the grep command above when checking for Chrome-related script references or debugging Chrome integration issues.
+
+## Architecture Notes
+
+### Crawl Model (0.9.x)
+- Crawl groups multiple Snapshots from a single `add` command
+- Each `add` creates one Crawl with one or more Snapshots
+- Seed model was removed - crawls now store URLs directly
+
+## Code Coverage
+
+### Overview
+
+Coverage tracking is enabled for passive collection across all contexts:
+- Unit tests (pytest)
+- Integration tests
+- Dev server (manual testing)
+- CLI usage
+
+Coverage data accumulates in `.coverage` file and can be viewed/analyzed to find dead code.
+
+### Install Coverage Tools
+
+```bash
+uv sync --dev  # Installs pytest-cov and coverage
+```
+
+### Running with Coverage
+
+#### Unit Tests
+```bash
+# Run tests with coverage
+pytest --cov=archivebox --cov-report=term archivebox/tests/
+
+# Or run specific test file
+pytest --cov=archivebox --cov-report=term archivebox/tests/test_migrations_08_to_09.py
+```
+
+#### Dev Server with Coverage
+```bash
+# Start dev server with coverage tracking
+coverage run --parallel-mode -m archivebox server
+
+# Or CLI commands
+coverage run --parallel-mode -m archivebox init
+coverage run --parallel-mode -m archivebox add https://example.com
+```
+
+#### Manual Testing (Always-On)
+To enable coverage during ALL Python executions (passive tracking):
+
+```bash
+# Option 1: Use coverage run wrapper
+coverage run --parallel-mode -m archivebox [command]
+
+# Option 2: Set environment variable (tracks everything)
+export COVERAGE_PROCESS_START=pyproject.toml
+# Now all Python processes will track coverage
+archivebox server
+archivebox add https://example.com
+```
+
+### Viewing Coverage
+
+#### Text Report (Quick View)
+```bash
+# Combine all parallel coverage data
+coverage combine
+
+# View summary
+coverage report
+
+# View detailed report with missing lines
+coverage report --show-missing
+
+# View specific file
+coverage report --include="archivebox/core/models.py" --show-missing
+```
+
+#### JSON Report (LLM-Friendly)
+```bash
+# Generate JSON report
+coverage json
+
+# View the JSON
+cat coverage.json | jq '.files | keys'  # List all files
+
+# Find files with low coverage
+cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 50) | "\(.key): \(.value.summary.percent_covered)%"'
+
+# Find completely uncovered files (dead code candidates)
+cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
+
+# Get missing lines for a specific file
+cat coverage.json | jq '.files["archivebox/core/models.py"].missing_lines'
+```
+
+#### HTML Report (Visual)
+```bash
+# Generate interactive HTML report
+coverage html
+
+# Open in browser
+open htmlcov/index.html
+```
+
+### Isolated Runs
+
+To measure coverage for specific scenarios:
+
+```bash
+# 1. Reset coverage data
+coverage erase
+
+# 2. Run your isolated test/scenario
+pytest --cov=archivebox archivebox/tests/test_migrations_fresh.py
+# OR
+coverage run --parallel-mode -m archivebox add https://example.com
+
+# 3. View results
+coverage combine
+coverage report --show-missing
+
+# 4. Optionally export for analysis
+coverage json
+```
+
+### Finding Dead Code
+
+```bash
+# 1. Run comprehensive tests + manual testing to build coverage
+pytest --cov=archivebox archivebox/tests/
+coverage run --parallel-mode -m archivebox server  # Use the app manually
+coverage combine
+
+# 2. Find files with 0% coverage (strong dead code candidates)
+coverage json
+cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
+
+# 3. Find files with <10% coverage (likely dead code)
+cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 10) | "\(.key): \(.value.summary.percent_covered)%"' | sort -t: -k2 -n
+
+# 4. Generate detailed report for analysis
+coverage report --show-missing > coverage_report.txt
+```
+
+### Tips
+
+- **Parallel mode** (`--parallel-mode`): Allows multiple processes to track coverage simultaneously without conflicts
+- **Combine**: Always run `coverage combine` before viewing reports to merge parallel data
+- **Reset**: Use `coverage erase` to start fresh for isolated measurements
+- **Branch coverage**: Enabled by default - tracks if both branches of if/else are executed
+- **Exclude patterns**: Config in `pyproject.toml` excludes tests, migrations, type stubs
+
+## Debugging Tips
+
+### Check Migration State
+```bash
+sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
+```
+
+### Check Table Schema
+```bash
+sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
+```
+
+### Verbose Test Output
+```bash
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
+```
+
+### Kill Zombie Chrome Processes
+```bash
+./bin/kill_chrome.sh
+```
@@ -1,24 +1,34 @@
-# This is the Dockerfile for ArchiveBox, it bundles the following dependencies:
-#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, yt-dlp, single-file
+# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
+#     python3.13, uv, python3-ldap
+#     curl, wget, git, dig, ping, tree, nano
+#     node, npm, single-file, readability-extractor, postlight-parser
+#     ArchiveBox, yt-dlp, playwright, chromium
 # Usage:
-#     git submodule update --init --recursive
-#     git pull --recurse-submodules
-#     docker build . -t archivebox --no-cache
+#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
+#     docker build . -t archivebox
 #     docker run -v "$PWD/data":/data archivebox init
 #     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
 #     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
 #     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
 # Multi-arch build:
 #     docker buildx create --use
-#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:0.7.5 -t archivebox/archivebox:dev
-#
-# Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
+#     docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
+# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+

 #########################################################################################

-# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
-FROM python:3.11-slim-bookworm
-# FROM debian:bookworm-backports
+### Example: Using ArchiveBox in your own project's Dockerfile ########
+
+# FROM python:3.13-slim
+# WORKDIR /data
+# RUN pip install archivebox>=0.9.0   # use latest release here
+# RUN archivebox install
+# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
+
+#########################################################################################
+
+FROM ubuntu:24.04

 LABEL name="archivebox" \
    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -44,10 +54,9 @@ ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
 ARG TARGETVARIANT
-
 ######### Environment Variables #################################

-# Global system-level config
+# Global build-time and runtime environment constants + default pkg manager config
 ENV TZ=UTC \
    LANGUAGE=en_US:en \
    LC_ALL=C.UTF-8 \
@@ -59,40 +68,38 @@ ENV TZ=UTC \
    PIP_DISABLE_PIP_VERSION_CHECK=1 \
    npm_config_loglevel=error

-# Version config
-ENV PYTHON_VERSION=3.11 \
+# Language Version config
+ENV PYTHON_VERSION=3.13 \
    NODE_VERSION=22

-# User config
+# Non-root User config
 ENV ARCHIVEBOX_USER="archivebox" \
    DEFAULT_PUID=911 \
    DEFAULT_PGID=911 \
    IN_DOCKER=True

-# Global paths
+# ArchiveBox Source Code + Lib + Data paths
 ENV CODE_DIR=/app \
    DATA_DIR=/data \
    PLAYWRIGHT_BROWSERS_PATH=/browsers
-    # GLOBAL_VENV=/venv \
-    # TODO: add TMP_DIR and LIB_DIR?

-# Build shell config
+# Bash SHELL config
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 

 ######### System Environment ####################################

-# Detect ArchiveBox version number by reading pyproject.toml
-COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
-RUN grep '^version = ' "${CODE_DIR}/pyproject.toml" | awk -F'"' '{print $2}' > /VERSION.txt
+# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
+WORKDIR "$CODE_DIR"

-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
+# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
 RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
    && rm -f /etc/apt/apt.conf.d/docker-clean

 # Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
+RUN (echo "[i] Docker build for ArchiveBox starting..." \
    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
    && echo \
@@ -114,16 +121,15 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
    | tee -a /VERSION.txt
-    # DEFAULT_PUID and DEFAULT_PID are overriden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
+    # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
    # https://docs.linuxserver.io/general/understanding-puid-and-pgid

 # Install system apt dependencies (adding backports to access more recent apt updates)
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
-    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' > /etc/apt/sources.list.d/backports.list \
+    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
    && mkdir -p /etc/apt/keyrings \
    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
+    && apt-get install -qq -y \
        # 1. packaging dependencies
        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
        # 2. docker and init system dependencies
@@ -133,6 +139,27 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
        # nano iputils-ping dnsutils htop procps jq yq
    && rm -rf /var/lib/apt/lists/*

+# Install apt binary dependencies for extractors
+# COPY --from=selenium/ffmpeg:latest /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing extractor dependencies for $TARGETPLATFORM..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        git ripgrep \
+        # Packages we have also needed in the past:
+        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
+        # curl wget (already installed above)
+    && rm -rf /var/lib/apt/lists/* \
+    # Save version info
+    && ( \
+        which curl && curl --version | head -n1 \
+        && which wget && wget --version 2>&1 | head -n1 \
+        && which git && git --version 2>&1 | head -n1 \
+        # && which ffmpeg && (ffmpeg --version 2>&1 | head -n1) || true \
+        && which rg && rg --version 2>&1 | head -n1 \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
 # Install sonic search backend
 COPY --from=archivebox/sonic:1.4.9 /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
@@ -140,12 +167,13 @@ RUN (which sonic && sonic --version) | tee -a /VERSION.txt

 ######### Language Environments ####################################

-# Install Python environment
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
-    # NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
+# Set up Python environment
+# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
+#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
    # && apt-get update -qq \
-    # && apt-get install -qq -y -t bookworm-backports --no-upgrade \
+    # && apt-get install -qq -y --no-upgrade \
    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
    # && rm -rf /var/lib/apt/lists/* \
    # tell PDM to allow using global system python site packages
@@ -158,22 +186,23 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
    # install global dependencies / python build dependencies in GLOBAL_VENV
    # && pip install --upgrade pip setuptools wheel \
    # Save version info
-    && ( \
-        which python3 && python3 --version | grep " $PYTHON_VERSION" \
-        && which pip && pip --version \
-        # && which pdm && pdm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
+    # && ( \
+    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
+    #     && which pip && pip --version \
+    #     # && which pdm && pdm --version \
+    #     && echo -e '\n\n' \
+    # ) | tee -a /VERSION.txt


-# Install Node environment
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Node $NODE_VERSION environment..." \
+# Set up Node environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing NODE $NODE_VERSION for $TARGETPLATFORM..." \
    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
-    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports --no-upgrade libatomic1 \
-    && apt-get install -y -t bookworm-backports --no-upgrade \
+    && apt-get install -qq -y --no-upgrade libatomic1 \
+    && apt-get install -y --no-upgrade \
        nodejs \
    && rm -rf /var/lib/apt/lists/* \
    # Update NPM to latest version
@@ -186,73 +215,106 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
    ) | tee -a /VERSION.txt


-
-######### Extractor Dependencies ##################################
-
-# Install apt dependencies
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing APT extractor dependencies globally using apt..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
-        curl wget git ffmpeg ripgrep \
-        # Packages we have also needed in the past:
-        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
-    && rm -rf /var/lib/apt/lists/* \
-    # Save version info
+# Set up uv and main app /venv
+RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
+ENV UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_PREFERENCE=managed \
+    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv
+WORKDIR "$CODE_DIR"
+# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
+    && uv venv /venv --python ${PYTHON_VERSION}
+ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
+RUN uv pip install setuptools pip \
    && ( \
-        which curl && curl --version | head -n1 \
-        && which wget && wget --version 2>&1 | head -n1 \
-        && which git && git --version 2>&1 | head -n1 \
-        && which rg && rg --version 2>&1 | head -n1 \
+        which python3 && python3 --version \
+        && which uv && uv self version \
+        && uv python find --system && uv python find \
        && echo -e '\n\n' \
    ) | tee -a /VERSION.txt


-# Install chromium browser using playwright
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
+######### ArchiveBox & Extractor Dependencies ##################################
+
+# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
-        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
-        at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
-        libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
-        libxaw7 libxcomposite1 libxdamage1 libxfont2 \
+    && apt-get install -qq -y --no-install-recommends \
+        build-essential gcc \
+        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
+        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
+    && uv pip install \
+        "python-ldap>=3.4.3" \
+    && apt-get purge -y \
+        python3-dev build-essential gcc \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+
+# Install apt font & rendering dependencies for chromium browser
+# TODO: figure out how much of this overlaps with `playwright install-deps chromium`
+# RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+
+# Install chromium browser binary using playwright
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing CHROMIUM dependencies, fonts, and display libraries for $TARGETPLATFORM..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y \
+        #fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+        #at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
+        #libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
+        #libxaw7 libxcomposite1 libxdamage1 libxfont2 \
        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \
        # xfonts-scalable xfonts-utils xserver-common xvfb \
        # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
        # libxss1 dbus dbus-x11 upower \
    # && service dbus start \
-    # install Chromium using playwright
-    && pip install playwright \
-    && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
-    && playwright install chromium \
-    && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
-    && rm -rf /var/lib/apt/lists/* \
+    && echo "[+] PIP Installing playwright into /venv and CHROMIUM binary into $PLAYWRIGHT_BROWSERS_PATH..." \
+    && uv pip install "playwright>=1.49.1" \
+    && uv run playwright install chromium --no-shell --with-deps \
+    && export CHROME_BINARY="$(uv run python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
    && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
+    && ln -s /browsers/ffmpeg-*/ffmpeg-linux /usr/bin/ffmpeg \
    && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/home/${ARCHIVEBOX_USER}/.config" \
    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
    && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
+    # delete extra full copy of node that playwright installs (saves >100mb)
+    && rm -f /venv/lib/python$PYTHON_VERSION/site-packages/playwright/driver/node \
    # Save version info
+    && rm -rf /var/lib/apt/lists/* \
    && ( \
-        which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
+        uv pip show playwright \
+        && which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
+        && which ffmpeg && ffmpeg -version \
        && echo -e '\n\n' \
    ) | tee -a /VERSION.txt

-# Install Node dependencies
+# Install Node extractor dependencies
 ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH"
 USER $ARCHIVEBOX_USER
 WORKDIR "/home/$ARCHIVEBOX_USER/.npm"
-RUN --mount=type=cache,target=/home/$ARCHIVEBOX_USER/.npm_cache,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT,uid=$DEFAULT_PUID,gid=$DEFAULT_PGID \
-    echo "[+] Installing NPM extractor dependencies in /home/$ARCHIVEBOX_USER/.npm..." \
+RUN --mount=type=cache,target=/home/archivebox/.npm_cache,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT,uid=$DEFAULT_PUID,gid=$DEFAULT_PGID \
+    echo "[+] NPM Installing node extractor dependencies into /home/$ARCHIVEBOX_USER/.npm..." \
    && npm config set prefix "/home/$ARCHIVEBOX_USER/.npm" \
    && npm install --global --prefer-offline --no-fund --no-audit --cache "/home/$ARCHIVEBOX_USER/.npm_cache" \
        "@postlight/parser@^2.2.3" \
        "readability-extractor@github:ArchiveBox/readability-extractor" \
        "single-file-cli@^1.1.54" \
        "puppeteer@^23.5.0" \
-        "@puppeteer/browsers@^2.4.0"
+        "@puppeteer/browsers@^2.4.0" \
+    && rm -Rf "/home/$ARCHIVEBOX_USER/.cache/puppeteer"
 USER root
+WORKDIR "$CODE_DIR"
 RUN ( \
        which node && node --version \
        && which npm && npm version \
@@ -265,69 +327,70 @@ RUN ( \

 ######### Build Dependencies ####################################

-# Install ArchiveBox Python dependencies
-WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR"/
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
-        build-essential gcc \
-        libssl-dev libldap2-dev libsasl2-dev \
-        python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-        pipx \
-    # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
-    # && pdm use --venv in-project \
-    # && pdm run python -m ensurepip \
-    # && pdm sync --fail-fast --no-editable --group :all --no-self \
-    # && pdm export -o requirements.txt --without-hashes \
-    # && source $GLOBAL_VENV/bin/activate \
-    && pip install -r requirements.txt \
-    && apt-get purge -y \
-        build-essential gcc \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*

-# Install ArchiveBox Python package from source
+# Install ArchiveBox Python venv dependencies from uv.lock
+RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    --mount=type=bind,source=uv.lock,target=/app/uv.lock \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml and uv.lock..." \
+    && uv sync \
+        --frozen \
+        --inexact \
+        --all-extras \
+        --no-install-project \
+        --no-install-workspace
+    # installs the pip packages that archivebox depends on, defined in pyproject.toml and uv.lock dependencies
+
+# Install ArchiveBox Python package + workspace dependencies from source
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
-    && pip install -e "${CODE_DIR}[all]" \
-    && rm -rf /var/lib/apt/lists/*
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && pip install \
+        --no-deps \
+        "$CODE_DIR" \
+    && ( \
+        pip show archivebox \
+        && which archivebox \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces

 ####################################################

 # Setup ArchiveBox runtime config
+ENV TMP_DIR=/tmp/archivebox \
+    LIB_DIR=/usr/share/archivebox/lib \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no
+
 WORKDIR "$DATA_DIR"
 RUN openssl rand -hex 16 > /etc/machine-id \
-    && mkdir -p "/tmp/archivebox" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/tmp/archivebox" \
-    && mkdir -p "/usr/share/archivebox/lib" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/usr/share/archivebox/lib"
-
-ENV GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no \
-    TMP_DIR=/tmp/archivebox \
-    LIB_DIR=/usr/share/archivebox/lib \
-    ALLOWED_HOSTS=*
+    && mkdir -p "$TMP_DIR" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
+    && mkdir -p "$LIB_DIR" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
+    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt

 # Print version for nice docker finish summary
-RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
+RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
    ) | tee -a /VERSION.txt
-RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt
+
+# Verify ArchiveBox is installed and print version info
+RUN chmod +x "$CODE_DIR"/bin/*.sh \
+    && gosu "$DEFAULT_PUID" archivebox version 2>&1 | tee -a /VERSION.txt || true

 ####################################################

-# Open up the interfaces to the outside world
+# Expose ArchiveBox's main interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000

 HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --silent 'http://localhost:8000/health/' | grep -q 'OK'
+    CMD curl --silent 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'

 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]
+CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]
@@ -1,21 +0,0 @@
-# Example: Using ArchiveBox in your own project
-
-FROM python:3.12-slim
-
-WORKDIR /app
-
-RUN pip install archivebox==0.8.5rc50
-RUN archivebox install
-
-RUN useradd -ms /bin/bash archivebox && chown -R archivebox /app
-
-
-
-
-
-
-
-
-
-
-
@@ -20,16 +20,18 @@ curl -fsSL 'https://get.archivebox.io' | bash    # (or see pip/brew/Docker instr
 <hr/>
 <br/>

-**ArchiveBox is a powerful, self-hosted internet archiving solution to collect, save, and view websites offline.**
+**ArchiveBox is a self-hosted app that lets you preserve content from websites in a variety of formats.**

-Without active preservation effort, everything on the internet eventually disappears or degrades. Archive.org does a great job as a centralized service, but saved URLs have to be public, and they can't save every type of content.
+We aim to make your data immediately useful, and kept in formats that other programs can read directly. As output, we save standard HTML, PNG, PDF, TXT, JSON, WARC, SQLite, all guaranteed to be readable for decades to come. ArchiveBox also has a CLI, REST API, and webhooks so you can set up integrations with other services.
+
+Without active preservation effort, everything on the internet eventually disappears or degrades.

 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>

 > ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  

-*Once installed, it can be used as a [CLI tool](#usage), [self-hosted Web App](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python library](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [one-off command](#static-archive-exporting).*
+*Once installed, you can interact with it through the: [Browser Extension](https://github.com/ArchiveBox/archivebox-browser-extension), [CLI](#usage), [self-hosted web interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python API](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [filesystem](#static-archive-exporting).*

 <br/>
 <hr/>
@@ -75,7 +77,7 @@ The goal is to sleep soundly knowing the part of the internet you care about wil
 <pre lang="bash"><code style="white-space: pre-line"># Option A: Get ArchiveBox with Docker Compose (recommended):
 mkdir -p ~/archivebox/data && cd ~/archivebox
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml   # edit options in this file as-needed
-docker compose run archivebox init --setup
+docker compose run archivebox init --install
 # docker compose run archivebox add 'https://example.com'
 # docker compose run archivebox help
 # docker compose up
@@ -83,7 +85,7 @@ docker compose run archivebox init --setup
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -it -v $PWD:/data archivebox/archivebox init --setup
+docker run -it -v $PWD:/data archivebox/archivebox init --install
 # docker run -it -v $PWD:/data archivebox/archivebox add 'https://example.com'
 # docker run -it -v $PWD:/data archivebox/archivebox help
 # docker run -it -v $PWD:/data -p 8000:8000 archivebox/archivebox
@@ -92,7 +94,7 @@ docker run -it -v $PWD:/data archivebox/archivebox init --setup
 # Option C: Or install it with your preferred pkg manager (see Quickstart below for apt, brew, and more)
 pip install archivebox
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --setup
+archivebox init --install
 # archivebox add 'https://example.com'
 # archivebox help
 # archivebox server 0.0.0.0:8000
@@ -102,7 +104,8 @@ archivebox init --setup
 curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
-<sub>Open <a href="http://localhost:8000"><code>http://localhost:8000</code></a> to see your server's Web UI ➡️</sub>
+<sub>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sub><br/>
+<sub>Set <code>LISTEN_HOST</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sub>
 </details>
 <br/>

@@ -130,7 +133,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
- [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
 - [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
 - Advanced users: support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
 - Planned: support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
@@ -150,9 +153,9 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 - **Individuals:**
  `saving bookmarks`, `preserving portfolio content`, `legacy / memoirs archival`
 - **Governments:**
-  `snapshoting public service sites`, `recordkeeping compliance`
+  `snapshotting public service sites`, `recordkeeping compliance`

-> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.*  (we are also seeking [grant funding](https://github.com/ArchiveBox/ArchiveBox/issues/1126#issuecomment-1487431394))  
+> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.*  
 > We offer: setup & support, CAPTCHA/ratelimit unblocking, SSO, audit logging/chain-of-custody, and more  
 > *ArchiveBox is a 🏛️ 501(c)(3) [nonprofit FSP](https://hackclub.com/hcb/) and all our work supports open-source development.* 

@@ -186,7 +189,7 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
 </code></pre></li>
 <li>Run the initial setup to create an admin user (or set ADMIN_USER/PASS in docker-compose.yml)
-<pre lang="bash"><code style="white-space: pre-line">docker compose run archivebox init --setup
+<pre lang="bash"><code style="white-space: pre-line">docker compose run archivebox init --install
 </code></pre></li>
 <li>Next steps: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">docker compose up
@@ -210,7 +213,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -v $PWD:/data -it archivebox/archivebox init --setup
+docker run -v $PWD:/data -it archivebox/archivebox init --install
 </code></pre>
 </li>
 <li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
@@ -256,19 +259,18 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a
 <br/>
 <ol>

-<li>Install <a href="https://realpython.com/installing-python/">Python >= v3.10</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v18</a> on your system (if not already installed).</li>
-<li>Install the ArchiveBox package using <code>pip3</code> (or <a href="https://pipx.pypa.io"><code>pipx</code></a>).
-<pre lang="bash"><code style="white-space: pre-line">pip3 install --upgrade archivebox yt-dlp playwright
-playwright install --with-deps chromium
+<li>Install <a href="https://realpython.com/installing-python/">Python >= v3.13</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v22</a> on your system (if not already installed).</li>
+<li>Install the ArchiveBox package using <code>pip3</code> (or <a href="https://docs.astral.sh/uv/guides/tools/#running-tools"><code>uvx</code></a>).
+<pre lang="bash"><code style="white-space: pre-line">pip3 install --upgrade archivebox
 archivebox version
 # install any missing extras shown using apt/brew/pkg/etc. see Wiki for instructions
-#    python@3.10 node curl wget git ripgrep ...
+#    python@3.13 node curl wget git ripgrep ...
 </code></pre>
 <i>See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install">Install: Bare Metal</a> Wiki for full install instructions for each OS...</i>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data   # for example
-archivebox init --setup   # instantialize a new collection
+archivebox init --install   # instantialize a new collection
 # (--setup auto-installs and link JS dependencies: singlefile, readability, mercury, etc.)
 </code></pre>
 </li>
@@ -291,25 +293,19 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <details>
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
-See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup">Install: Bare Metal</a> Wiki for instructions. ➡️
-<!--<ol>
-<li>Add the ArchiveBox repository to your sources.<br/>
-<pre lang="bash"><code style="white-space: pre-line">echo "deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee /etc/apt/sources.list.d/archivebox.list
-sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C258F79DCC02E369
-sudo apt update
-</code></pre>
-</li>
-<li>Install the ArchiveBox package using <code>apt</code>.
-<pre lang="bash"><code style="white-space: pre-line">sudo apt install archivebox
-# update to newest version with pip (sometimes apt package is outdated)
-pip install --upgrade --ignore-installed archivebox yt-dlp playwright
-playwright install --with-deps chromium    # install chromium and its system dependencies
+<ol>
+<li>Download and install the <code>.deb</code> package from the <a href="https://github.com/ArchiveBox/ArchiveBox/releases">latest release</a>.
+<pre lang="bash"><code style="white-space: pre-line"># download the .deb for your architecture (amd64 or arm64)
+ARCH="$(dpkg --print-architecture)"
+VERSION="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'].lstrip('v'))")"
+curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${VERSION}_${ARCH}.deb" -o /tmp/archivebox.deb
+sudo apt install /tmp/archivebox.deb
 archivebox version                         # make sure all dependencies are installed
 </code></pre>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --setup
+archivebox init --install
 </code></pre>
 <br/>
 </li>
@@ -322,7 +318,7 @@ archivebox help
 </li>
 </ol>
 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
-<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>-->
+<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>
 <br/><br/>
 </details>

@@ -334,16 +330,13 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <li>Install the ArchiveBox package using <code>brew</code>.
 <pre lang="bash"><code style="white-space: pre-line">brew tap archivebox/archivebox
 brew install archivebox
-# update to newest version with pip (sometimes brew package is outdated)
-pip install --upgrade --ignore-installed archivebox yt-dlp playwright
-playwright install --with-deps chromium    # install chromium and its system dependencies
 archivebox version                         # make sure all dependencies are installed
 </code></pre>
 <i>See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup">Install: Bare Metal</a> Wiki for more granular instructions for macOS... ➡️</i>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --setup
+archivebox init --install
 </code></pre>
 </li>
 <li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
@@ -367,7 +360,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 > *Warning: These are contributed by external volunteers and may lag behind the official `pip` channel.*

 <ul>
-<li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>)</li>
+<li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>, maintained by <a href="https://github.com/jasongodev"><code>@jasongodev</code></a>)</li>
 <li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>pkg</code> + <code>pip3</code> under-the-hood)</li>
 <li>Nix: <a href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/archivebox/default.nix"><code>nix-env --install archivebox</code></a> (contributed by <a href="https://github.com/siraben"><code>@siraben</code></a>)</li>
 <li>Guix: <a href="https://packages.guix.gnu.org/packages/archivebox/"><code>guix install archivebox</code></a> (contributed by <a href="https://github.com/rakino"><code>@rakino</code></a>)</li>
@@ -409,6 +402,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 <ul>
 <li><s>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></s> (<a href="https://truecharts.org/news/scale-deprecation/">TrueCharts is discontinued</a>, wait for <a href="https://forums.truenas.com/t/the-future-of-electric-eel-and-apps/5409/">Electric Eel</a>)</li>
 <li><a href="https://unraid.net/community/apps?q=archivebox#r">UnRaid</a></li>
+<li><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a></li>
 <li><a href="https://github.com/YunoHost-Apps/archivebox_ynh">Yunohost</a></li>
 <li><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a></li>
 <li><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a></li>
@@ -429,7 +423,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 <ul>
 <li><a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">
 <img src="https://img.shields.io/badge/Custom_Development-ArchiveBox.io-%231a1a1a.svg?style=flat" height="22px"/>
-</a> (<a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">get hosting, support, and feature customization directy from us</a>)</li>
+</a> (<a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">get hosting, support, and feature customization directly from us</a>)</li>
 <li><a href="https://monadical.com">
 <img src="https://img.shields.io/badge/General_Dev_Consulting-Monadical.com-%231a1a1a.svg?style=flat" height="22px"/>
 </a> (<a href="https://monadical.com/contact-us.html">generalist consultancy that has ArchiveBox experience</a>)</li>
@@ -466,6 +460,7 @@ For more discussion on managed and paid hosting options see here: <a href="https
 #### ➡️&nbsp; Next Steps

 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
+- (Optional) Create a persona and import browser cookies to archive logged-in sites: `archivebox persona create --import=chrome personal`
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -492,13 +487,19 @@ docker compose run archivebox help

 # equivalent: docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
 docker run -it -v $PWD:/data archivebox/archivebox help
+
+# optional: import your browser cookies into a persona for logged-in archiving
+archivebox persona create --import=chrome personal
+# supported: chrome/chromium/brave/edge (Chromium-based only)
+# use --profile to target a specific profile (e.g. Default, Profile 1)
+# re-running import merges/dedupes cookies.txt (by domain/path/name) but replaces chrome_user_data
 ```

 #### ArchiveBox Subcommands

 - `archivebox` `help`/`version` to see the list of available subcommands / currently installed version info
 - `archivebox` `setup`/`init`/`config`/`status`/`shell`/`manage` to administer your collection
- `archivebox` `add`/`oneshot`/`schedule` to pull in fresh URLs from [bookmarks/history/RSS/etc.](#input-formats)
+- `archivebox` `add`/`schedule` to pull in fresh URLs from [bookmarks/history/RSS/etc.](#input-formats)
 - `archivebox` `list`/`update`/`remove` to manage existing Snapshots in your collection

 <br/>
@@ -509,7 +510,7 @@ docker run -it -v $PWD:/data archivebox/archivebox help
 # make sure you have pip-installed ArchiveBox and it's available in your $PATH first  
 <br/>
 # archivebox [subcommand] [--help]
-archivebox init --setup      # safe to run init multiple times (also how you update versions)
+archivebox init --install      # safe to run init multiple times (also how you update versions)
 archivebox version           # get archivebox version info + check dependencies
 archivebox help              # get list of archivebox subcommands that can be run
 archivebox add --depth=1 'https://news.ycombinator.com'
@@ -526,7 +527,7 @@ archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
 # docker compose run archivebox [subcommand] [--help]
-docker compose run archivebox init --setup
+docker compose run archivebox init --install
 docker compose run archivebox version
 docker compose run archivebox help
 docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
@@ -544,7 +545,7 @@ docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you create and cd into in a new empty directory first  
 <br/>
 # docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox init --setup
+docker run -v $PWD:/data -it archivebox/archivebox init --install
 docker run -v $PWD:/data -it archivebox/archivebox version
 docker run -v $PWD:/data -it archivebox/archivebox help
 docker run -v $PWD:/data -it archivebox/archivebox add --depth=1 'https://news.ycombinator.com'
@@ -584,7 +585,8 @@ docker run -v $PWD:/data -it archivebox/archivebox archivebox manage createsuper
 docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 </code></pre>

-<sup>Open <a href="http://localhost:8000"><code>http://localhost:8000</code></a> to see your server's Web UI ➡️</sup>
+<sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
+<sup>Set <code>LISTEN_HOST</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sup>
 <br/><br/>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage">Usage: Web UI</a> wiki. ➡️</i>
 <br/><br/>
@@ -688,7 +690,7 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a

 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.

-It also includes a built-in scheduled import feature with `archivebox schedule` and browser bookmarklet, so you can pull in URLs from RSS feeds, websites, or the filesystem regularly/on-demand.
+It also includes a built-in scheduled import feature with `archivebox schedule`, handled by the same orchestrator that powers `archivebox server`, so you can pull in URLs from RSS feeds and websites regularly without a separate cron container.

 <br/>

@@ -749,7 +751,7 @@ env CHROME_BINARY=chromium archivebox ...       # run with a one-off config
 <sub>These methods also work the same way when run inside Docker, see the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Docker Configuration</a> wiki page for details.</sub>
 </details><br/>

-The configuration is documented here: **[Configuration Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)**, and loaded here: [`archivebox/config.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config.py).
+The configuration is documented here: **[Configuration Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)**, and loaded from: [`archivebox/config/`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config/).

 <a name="most-common-options-to-tweak"></a>
 <details>
@@ -760,16 +762,12 @@ The configuration is documented here: **[Configuration Wiki](https://github.com/
 <br/>
 TIMEOUT=240                # default: 60    add more seconds on slower networks
 CHECK_SSL_VALIDITY=False   # default: True  False = allow saving URLs w/ bad SSL
-SAVE_ARCHIVE_DOT_ORG=False # default: True  False = disable Archive.org saving
-MAX_MEDIA_SIZE=1500m       # default: 750m  raise/lower youtubedl output size
 <br/>
 PUBLIC_INDEX=True          # default: True  whether anon users can view index
 PUBLIC_SNAPSHOTS=True      # default: True  whether anon users can view pages
 PUBLIC_ADD_VIEW=False      # default: False whether anon users can add new URLs
 <br/>
-CHROME_USER_AGENT="Mozilla/5.0 ..."  # change these to get around bot blocking
-WGET_USER_AGENT="Mozilla/5.0 ..."
-CURL_USER_AGENT="Mozilla/5.0 ..."
+USER_AGENT="Mozilla/5.0 ..."  # change this to get around bot blocking
 </code></pre>
 </details>
 <br/>
@@ -791,14 +789,14 @@ ArchiveBox bundles industry-standard tools like [Google Chrome](https://github.c
 </blockquote>

 <ul>
-<li>Language: Python <code>&gt;=3.10</code></li>
+<li>Language: Python <code>&gt;=3.13</code></li>
 <li>Backend: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://django-ninja.dev/">Django-Ninja</a> for REST API</li>
-<li>Frontend: <a href="https://docs.djangoproject.com/en/5.1/ref/contrib/admin/">Django Admin</a> + Vanilla HTML, CSS, JS</li>
-<li>Web Server: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://channels.readthedocs.io/en/latest/"><code>channels</code></a> + <a href="https://github.com/django/daphne/"><code>daphne]</code></a></li>
-<li>Database: <a href="https://docs.djangoproject.com/en/5.1/ref/databases/#sqlite-notes">Django ORM</a> saving to <a href="https://www.sqlite.org/mostdeployed.html">SQLite3</a> <code>./data/index.sqlite</code></li>
-<li>Job Queue: <a href="https://huey.readthedocs.io/">Huey</a> using <code>./data/queue.sqlite3</code> under <code>supervisord</code></li>
-<li>Build/test/lint: <a href="https://github.com/pdm-project/pdm"><code>pdm</code></a> / <code>mypy</code>+<code>pyright</code>+<code>pytest</code> / <code>ruff</code></li>
-<li>Subdependencies: <a href="https://github.com/ArchiveBox/pydantic-pkgr"><code>pydantic-pkgr</code></a> installs apt/brew/pip/npm pkgs at runtime (e.g. <code>yt-dlp</code>, <code>singlefile</code>, <code>readability</code>, <code>git</code>)</li>
+<li>Frontend: <a href="https://docs.djangoproject.com/en/6.0/ref/contrib/admin/">Django Admin</a> + Vanilla HTML, CSS, JS</li>
+<li>Web Server: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://github.com/django/daphne/"><code>daphne</code></a> (ASGI)</li>
+<li>Database: <a href="https://docs.djangoproject.com/en/6.0/ref/databases/#sqlite-notes">Django ORM</a> saving to <a href="https://www.sqlite.org/mostdeployed.html">SQLite3</a> <code>./data/index.sqlite3</code></li>
+<li>Job Queue: Custom orchestrator using <code>supervisord</code> for worker management</li>
+<li>Build/test/lint: <a href="https://github.com/astral-sh/uv"><code>uv</code></a> / <code>pyright</code>+<code>ty</code>+<code>pytest</code> / <code>ruff</code></li>
+<li>Subdependencies: <a href="https://github.com/ArchiveBox/abx-pkg"><code>abx-pkg</code></a> installs apt/brew/pip/npm pkgs at runtime (e.g. <code>yt-dlp</code>, <code>singlefile</code>, <code>readability</code>, <code>git</code>)</li>
 </ul>


@@ -827,7 +825,7 @@ If not using Docker, make sure to keep the dependencies up-to-date yourself and
 # apt/brew/pip/etc install ... (see Quickstart instructions above)
 <br/>
 which -a archivebox    # see where you have installed archivebox
-archivebox setup       # auto install all the extractors and extras
+archivebox install     # auto install all the extractors and extras
 archivebox --version   # see info and check validity of installed dependencies
 </code></pre>
  
@@ -897,7 +895,7 @@ Each snapshot subfolder <code>data/archive/TIMESTAMP/</code> includes a static <

 ## Static Archive Exporting

-You can create one-off archives of individual URLs with `archivebox oneshot`, or export your index as static HTML using `archivebox list` (so you can view it without an ArchiveBox server).
+You can export your index as static HTML using `archivebox list` (so you can view it without an ArchiveBox server).

 <br/>
 <details>
@@ -907,10 +905,7 @@ You can create one-off archives of individual URLs with `archivebox oneshot`, or
 <p><em>NOTE: These exports are not paginated, exporting many URLs or the entire archive at once may be slow. Use the filtering CLI flags on the <code>archivebox list</code> command to export specific Snapshots or ranges.</em></p>
 </blockquote>

-<pre lang="bash"><code style="white-space: pre-line"># do a one-off single URL archive wihout needing a data dir initialized
-archivebox oneshot 'https://example.com'
-
-# archivebox list --help
+<pre lang="bash"><code style="white-space: pre-line"># archivebox list --help
 archivebox list --html --with-headers > index.html     # export to static html table
 archivebox list --json --with-headers > index.json     # export to json blob
 archivebox list --csv=timestamp,url,title > index.csv  # export to csv spreadsheet
@@ -955,18 +950,11 @@ If you're importing pages with private content or URLs containing secret tokens
 archivebox add 'https://docs.google.com/document/d/12345somePrivateDocument'
 archivebox add 'https://vimeo.com/somePrivateVideo'

-# without first disabling saving to Archive.org:
-archivebox config --set SAVE_ARCHIVE_DOT_ORG=False  # disable saving all URLs in Archive.org
-
 # restrict the main index, Snapshot content, and Add Page to authenticated users as-needed:
 archivebox config --set PUBLIC_INDEX=False
 archivebox config --set PUBLIC_SNAPSHOTS=False
-archivebox config --set PUBLIC_ADD_VIEW=False 
+archivebox config --set PUBLIC_ADD_VIEW=False
 archivebox manage createsuperuser
-
-# if extra paranoid or anti-Google:
-archivebox config --set SAVE_FAVICON=False          # disable favicon fetching (it calls a Google API passing the URL's domain part only)
-archivebox config --set CHROME_BINARY=chromium      # ensure it's using Chromium instead of Chrome
 </code></pre>

 <blockquote>
@@ -1089,7 +1077,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <br/>

 <ul>
-<li><strong>ArchiveBox can use anywhere from ~1gb per 1000 Snapshots, to ~50gb per 1000 Snapshots</strong>, mostly dependent on whether you're saving audio & video using <code>SAVE_MEDIA=True</code> and whether you lower <code>MEDIA_MAX_SIZE=750mb</code>.</li>
+<li><strong>ArchiveBox can use anywhere from ~1gb per 1000 Snapshots, to ~50gb per 1000 Snapshots</strong>, mostly dependent on whether you're saving video/audio using <code>YTDLP_ENABLED=True</code> and whether you lower <code>YTDLP_MAX_SIZE=750m</code>.</li>
 <li>Disk usage can be reduced by using a compressed/<a href="https://www.ixsystems.com/blog/ixsystems-and-klara-systems-celebrate-valentines-day-with-a-heartfelt-donation-of-fast-dedupe-to-openzfs-and-truenas/">deduplicated</a> filesystem like <a href="https://www.reddit.com/r/zfs/comments/t9cexx/a_simple_real_world_zfs_compression_speed_an/">ZFS</a>/BTRFS, or by turning off extractors methods you don't need. You can also deduplicate content with a tool like <a href="https://github.com/adrianlopezroche/fdupes"><code>fdupes</code></a> or <a href="https://github.com/pauldreik/rdfind"><code>rdfind</code></a>.  
 </li>
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
@@ -1195,7 +1183,7 @@ ArchiveBox's stance is that duplication of other people's content is only ethica

 In the U.S., <a href="https://guides.library.oregonstate.edu/copyright/libraries">libraries, researchers, and archivists</a> are allowed to duplicate copyrighted materials under <a href="https://libguides.ala.org/copyright/fairuse">"fair use"</a> for <a href="https://guides.cuny.edu/cunyfairuse/librarians#:~:text=One%20of%20these%20specified%20conditions,may%20be%20liable%20for%20copyright">private study, scholarship, or research</a>. Archive.org's non-profit preservation work is <a href="https://blog.archive.org/2024/03/01/fair-use-in-action-at-the-internet-archive/">covered under fair use</a> in the US, and they properly handle <a href="https://cardozoaelj.com/2015/03/20/use-of-copyright-law-to-take-down-revenge-porn/">unethical content</a>/<a href="https://help.archive.org/help/rights/">DMCA</a>/<a href="https://gdpr.eu/right-to-be-forgotten/#:~:text=An%20individual%20has%20the%20right,that%20individual%20withdraws%20their%20consent.">GDPR</a> removal requests to maintain good standing in the eyes of the law.

-As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use sofware like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal council if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and changing your instance's branding using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#custom_templates_dir"><code>CUSTOM_TEMPLATES_DIR</code></a>).
+As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use software like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal council if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and changing your instance's branding using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#custom_templates_dir"><code>CUSTOM_TEMPLATES_DIR</code></a>).

 </details>
 <br/>
@@ -1224,7 +1212,7 @@ ArchiveBox tries to be a robust, set-and-forget archiving solution suitable for

 Not all content is suitable to be archived on a centralized, publicly accessible platform. Archive.org doesn't offer the ability to save things behind login walls for good reason, as the content may not have been intended for a public audience. ArchiveBox exists to fill that gap by letting everyone save what they have access to on an individual basis, and to encourage decentralized archiving that's less succeptible to censorship or natural disasters.

-By having users store their content locally or within their organizations, we can also save much larger portions of the internet than a centralized service has the disk capcity handle. The eventual goal is to work towards federated archiving where users can share portions of their collections with each other, and with central archives on a case-by-case basis.
+By having users store their content locally or within their organizations, we can also save much larger portions of the internet than a centralized service has the disk capacity to handle. The eventual goal is to work towards federated archiving where users can share portions of their collections with each other, and with central archives on a case-by-case basis.

 <h3>Comparison With Other Self-Hosted Archiving Options</h3>

@@ -1233,7 +1221,7 @@ ArchiveBox differentiates itself from [similar self-hosted projects](https://git

 *If you want better fidelity for very complex interactive pages with heavy JS/streams/API requests, check out [ArchiveWeb.page](https://archiveweb.page) and [ReplayWeb.page](https://replayweb.page).*

-*If you want more bookmark categorization and note-taking features, check out [Archivy](https://archivy.github.io/), [Memex](https://github.com/WorldBrain/Memex), [Polar](https://getpolarized.io/), or [LinkAce](https://www.linkace.org/).*
+*If you want more bookmark categorization and note-taking features, check out [Memex](https://github.com/WorldBrain/Memex), [Hoarder](https://github.com/hoarder-app/hoarder), [LinkWarden](https://github.com/linkwarden/linkwarden),  [Archivy](https://archivy.github.io/), or [LinkAce](https://www.linkace.org/).*

 *If you need more advanced recursive spider/crawling ability beyond `--depth=1`, check out [Browsertrix](https://github.com/webrecorder/browsertrix-crawler), [Photon](https://github.com/s0md3v/Photon), or [Scrapy](https://scrapy.org/) and pipe the outputted URLs into ArchiveBox.*

@@ -1349,8 +1337,6 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc

 **Internal Architecture Diagrams:** https://github.com/ArchiveBox/ArchiveBox/wiki/ArchiveBox-Architecture-Diagrams

-<img width="100%" alt="architecture diagrams" src="https://github.com/user-attachments/assets/bdf782d5-2c8f-4168-845d-6794ff5cbfec">
-

 ### Setup the dev environment

@@ -1371,11 +1357,18 @@ git pull --recurse-submodules
 ```bash
 # Install ArchiveBox + python dependencies
 pip install uv
-uv venv
-uv sync
+./bin/lock_pkgs.sh         # (aka `uv venv; uv sync;` + generate requirements.txt)
+source .venv/bin/activate  # activate the venv

-archivebox init
-archivebox setup
+# Install ArchiveBox runtime dependencies
+mkdir -p data && cd data
+archivebox install         # detect and install all extractor dependencies
+
+# Run the development server w/ autoreloading (but no bg workers)
+archivebox server --debug --reload 0.0.0.0:8000
+
+# Run the production server (with bg workers but no autoreloading)
+archivebox server 0.0.0.0:8000
 ```

 #### 2. Option B: Build the docker container and use that for development instead
@@ -1384,14 +1377,15 @@ archivebox setup
 # Optional: develop via docker by mounting the code dir into the container
 # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
 # inside the container will reload and pick up your changes
-docker build . -t archivebox
-docker run -it \
-    -v $PWD/data:/data \
-    archivebox init --setup
-docker run -it -p 8000:8000 \
-    -v $PWD/data:/data \
-    -v $PWD/archivebox:/app/archivebox \
-    archivebox server 0.0.0.0:8000 --debug --reload
+./bin/build_docker.sh dev
+
+docker run -it -v $PWD/data:/data archivebox/archivebox:dev init --install
+
+# Run the development server w/ autoreloading (but no bg workers)
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
+
+# Run the production server (with bg workers but no autoreloading)
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server

 # (remove the --reload flag and add the --nothreading flag when profiling with the django debug toolbar)
 # When using --reload, make sure any files you create can be read by the user in the Docker container, eg with 'chmod a+rX'.
@@ -1409,11 +1403,15 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 <details><summary><i>Click to expand...</i></summary>

 ```bash
+# set up persistent DEBUG=True for all runs
 archivebox config --set DEBUG=True
+
+# OR you can run a dev server with DEBUG=True in a few ways:
+archivebox server --debug --reload 0.0.0.0:8000
 # or
-archivebox server --debug ...
-# faster dev version wo/ bg workers enabled:
-daphne -b 0.0.0.0 -p 8000 archivebox.core.asgi:application
+archivebox server --debug 0.0.0.0:8000
+# or
+env DEBUG=True daphne -b 0.0.0.0 -p 8000 archivebox.core.asgi:application
 ```

 https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running
@@ -1466,7 +1464,7 @@ archivebox install
 ./bin/lint.sh
 ./bin/test.sh
 ```
-(uses `flake8`, `mypy`, and `pytest -s`)
+(uses `ruff`, `pyright`, `ty`, and `pytest -s`)

 </details>

@@ -1620,7 +1618,8 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 <a href="https://github.com/sponsors/pirate"><img src="https://img.shields.io/badge/Github_Sponsors-%23B7CDFE.svg"/></a> &nbsp;
 <a href="https://www.patreon.com/theSquashSH"><img src="https://img.shields.io/badge/Patreon-%23DD5D76.svg"/></a> &nbsp;
 <a href="https://paypal.me/NicholasSweeting"><img src="https://img.shields.io/badge/Paypal-%23FFD141.svg"/></a> &nbsp;
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"><img src="https://img.shields.io/badge/BTC%5CETH-%231a1a1a.svg"/></a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"><img src="https://img.shields.io/badge/BTC%5CETH-%231a1a1a.svg"/></a> &nbsp;
+<a href="https://archivebox-shop.fourthwall.com/"><img src="https://img.shields.io/badge/Merch-%23903851.svg"/></a>
 <br/>
 <sup><i>ArchiveBox operates as a US 501(c)(3) nonprofit <a href="https://en.wikipedia.org/wiki/Fiscal_sponsorship">FSP</a> (sponsored by <a href="https://hackclub.com/hcb?ref=donation">HCB</a>), <a href="https://hcb.hackclub.com/donations/start/archivebox">direct donations</a> are tax-deductible.</i></sup>
 <br/><br/>
@@ -9,12 +9,25 @@
 # in a universe that seems indifferent to us."
 # --Norber Weiner

-__package__ = 'archivebox'
+__package__ = "archivebox"

 import os
 import sys
 from pathlib import Path
-from typing import cast
+from typing import Protocol, cast
+
+from abx_plugins import get_plugins_dir
+
+
+class _ReconfigurableStream(Protocol):
+    def reconfigure(self, *, line_buffering: bool) -> object: ...
+
+
+# Force unbuffered output for real-time logs
+if hasattr(sys.stdout, "reconfigure"):
+    cast(_ReconfigurableStream, sys.stdout).reconfigure(line_buffering=True)
+    cast(_ReconfigurableStream, sys.stderr).reconfigure(line_buffering=True)
+os.environ["PYTHONUNBUFFERED"] = "1"

 ASCII_LOGO = """
 █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
@@ -25,83 +38,58 @@ ASCII_LOGO = """
 ╚═╝  ╚═╝╚═╝  ╚═╝ ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝  ╚══════╝ ╚═════╝  ╚═════╝ ╚═╝  ╚═╝
 """

-# make sure PACKAGE_DIR is in sys.path so we can import all subfolders
-# without necessarily waiting for django to load them thorugh INSTALLED_APPS
 PACKAGE_DIR = Path(__file__).resolve().parent
-if str(PACKAGE_DIR) not in sys.path:
-    sys.path.append(str(PACKAGE_DIR))
-os.environ['DJANGO_SETTINGS_MODULE'] = 'core.settings'
-os.environ['TZ'] = 'UTC'
+
+# # Add PACKAGE_DIR to sys.path - required for Django migrations to import models
+# # Migrations reference models like 'machine.Binary' which need to be importable
+# if str(PACKAGE_DIR) not in sys.path:
+#     sys.path.append(str(PACKAGE_DIR))
+
+os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
+os.environ["TZ"] = "UTC"

 # detect ArchiveBox user's UID/GID based on data dir ownership
-from .config.permissions import drop_privileges                 # noqa
+from .config.permissions import drop_privileges  # noqa
+
 drop_privileges()

-from .misc.checks import check_not_root, check_io_encoding      # noqa
+from .misc.checks import check_not_root, check_not_inside_source_dir, check_io_encoding  # noqa
+
 check_not_root()
+check_not_inside_source_dir()
 check_io_encoding()

-# print('INSTALLING MONKEY PATCHES')
-from .monkey_patches import *                    # noqa
-# print('DONE INSTALLING MONKEY PATCHES')
+# Install monkey patches for third-party libraries
+from .misc.monkey_patches import *  # noqa

+# Plugin directories
+BUILTIN_PLUGINS_DIR = Path(get_plugins_dir()).resolve()
+USER_PLUGINS_DIR = (
+    Path(
+        os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or os.environ.get("USER_PLUGINS_DIR") or os.environ.get("DATA_DIR", os.getcwd()),
+    )
+    / "custom_plugins"
+)

-# print('LOADING VENDORED LIBRARIES')
-from .pkgs import load_vendored_pkgs             # noqa
-load_vendored_pkgs()
-# print('DONE LOADING VENDORED LIBRARIES')
-
-# Load ABX Plugin Specifications + Default Implementations
-import abx                                       # noqa
-import abx_spec_archivebox                       # noqa
-import abx_spec_config                           # noqa
-import abx_spec_pydantic_pkgr                    # noqa
-import abx_spec_django                           # noqa
-import abx_spec_searchbackend                    # noqa
-
-abx.pm.add_hookspecs(abx_spec_config.PLUGIN_SPEC)
-abx.pm.register(abx_spec_config.PLUGIN_SPEC())
-
-abx.pm.add_hookspecs(abx_spec_pydantic_pkgr.PLUGIN_SPEC)
-abx.pm.register(abx_spec_pydantic_pkgr.PLUGIN_SPEC())
-
-abx.pm.add_hookspecs(abx_spec_django.PLUGIN_SPEC)
-abx.pm.register(abx_spec_django.PLUGIN_SPEC())
-
-abx.pm.add_hookspecs(abx_spec_searchbackend.PLUGIN_SPEC)
-abx.pm.register(abx_spec_searchbackend.PLUGIN_SPEC())
-
-# Cast to ArchiveBoxPluginSpec to enable static type checking of pm.hook.call() methods
-abx.pm = cast(abx.ABXPluginManager[abx_spec_archivebox.ArchiveBoxPluginSpec], abx.pm)
-pm = abx.pm
-
-
-# Load all pip-installed ABX-compatible plugins
-ABX_ECOSYSTEM_PLUGINS = abx.get_pip_installed_plugins(group='abx')
-
-# Load all built-in ArchiveBox plugins
-ARCHIVEBOX_BUILTIN_PLUGINS = {
-    'config': PACKAGE_DIR / 'config',
-    'core': PACKAGE_DIR / 'core',
-    # 'search': PACKAGE_DIR / 'search',
-    # 'core': PACKAGE_DIR / 'core',
+# These are kept for backwards compatibility with existing code
+# that checks for plugins. The new hook system uses discover_hooks()
+ALL_PLUGINS = {
+    "builtin": BUILTIN_PLUGINS_DIR,
+    "user": USER_PLUGINS_DIR,
 }
-
-# Load all user-defined ArchiveBox plugins
-USER_PLUGINS = abx.find_plugins_in_dir(Path(os.getcwd()) / 'user_plugins')
-
-# Import all plugins and register them with ABX Plugin Manager
-ALL_PLUGINS = {**ABX_ECOSYSTEM_PLUGINS, **ARCHIVEBOX_BUILTIN_PLUGINS, **USER_PLUGINS}
-LOADED_PLUGINS = abx.load_plugins(ALL_PLUGINS)
+LOADED_PLUGINS = ALL_PLUGINS

 # Setup basic config, constants, paths, and version
-from .config.constants import CONSTANTS                         # noqa
-from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR    # noqa
-from .config.version import VERSION                             # noqa
+from .config.constants import CONSTANTS  # noqa
+from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR  # noqa
+from .config.version import VERSION  # noqa
+
+# Set MACHINE_ID env var so hook scripts can use it
+os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)

 __version__ = VERSION
-__author__ = 'ArchiveBox'
-__license__ = 'MIT'
+__author__ = "ArchiveBox"
+__license__ = "MIT"

 ASCII_ICON = """
 ██████████████████████████████████████████████████████████████████████████████████████████████████ 
@@ -1,19 +1,20 @@
 #!/usr/bin/env python3
-"""This is the main entry point for the ArchiveBox CLI."""
-__package__ = 'archivebox'
+"""This is the entrypoint for python -m archivebox ..."""

-import archivebox      # noqa # make sure monkey patches are applied before anything else
+__package__ = "archivebox"
+
+import archivebox  # noqa # make sure monkey patches are applied before anything else
 import sys

 from .cli import main

 ASCII_LOGO_MINI = r"""
-     _             _     _           ____            
+     _             _     _           ____
    / \   _ __ ___| |__ (_)_   _____| __ )  _____  __
   / _ \ | '__/ __| '_ \| \ \ / / _ \  _ \ / _ \ \/ /
-  / ___ \| | | (__| | | | |\ V /  __/ |_) | (_) >  < 
+  / ___ \| | | (__| | | | |\ V /  __/ |_) | (_) >  <
 /_/   \_\_|  \___|_| |_|_| \_/ \___|____/ \___/_/\_\
 """

-if __name__ == '__main__':
+if __name__ == "__main__":
    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1 +0,0 @@
-__package__ = 'abid_utils'
@@ -1,223 +0,0 @@
-__package__ = 'archivebox.abid_utils'
-
-from typing import NamedTuple, Any, Union, Dict
-
-import ulid
-import uuid6
-import hashlib
-from urllib.parse import urlparse
-
-from uuid import UUID
-from typeid import TypeID            # type: ignore[import-untyped]
-from datetime import datetime
-
-from archivebox.misc.util import enforce_types
-
-
-ABID_PREFIX_LEN = 4
-ABID_SUFFIX_LEN = 26
-ABID_LEN = 30
-ABID_TS_LEN = 10
-ABID_URI_LEN = 8
-ABID_SUBTYPE_LEN = 2
-ABID_RAND_LEN = 6
-
-DEFAULT_ABID_PREFIX = 'obj_'
-
-# allows people to keep their uris secret on a per-instance basis by changing the salt.
-# the default means everyone can share the same namespace for URI hashes,
-# meaning anyone who has a URI and wants to check if you have it can guess the ABID
-DEFAULT_ABID_URI_SALT = '687c2fff14e3a7780faa5a40c237b19b5b51b089'
-
-
-class ABID(NamedTuple):
-    """
-    e.g. ABID('obj_01HX9FPYTRE4A5CCD901ZYEBQE')
-    """
-    prefix: str            # e.g. obj_
-    ts: str                # e.g. 01HX9FPYTR
-    uri: str               # e.g. E4A5CCD9
-    subtype: str           # e.g. 01
-    rand: str              # e.g. ZYEBQE
-    
-    # salt: str = DEFAULT_ABID_URI_SALT
-
-    def __getattr__(self, attr: str) -> Any:
-        return getattr(self.ulid, attr)
-
-    def __eq__(self, other: Any) -> bool:
-        try:
-            return self.ulid == other.ulid
-        except AttributeError:
-            return NotImplemented
-
-    def __str__(self) -> str:
-        return self.prefix + self.suffix
-
-    def __len__(self) -> int:
-        return len(self.prefix + self.suffix)
-
-    @classmethod
-    def parse(cls, buffer: Union[str, UUID, ulid.ULID, TypeID, 'ABID'], prefix=DEFAULT_ABID_PREFIX) -> 'ABID':
-        assert buffer, f'Attempted to create ABID from null value {buffer}'
-
-        buffer = str(buffer)
-        if '_' in buffer:
-            prefix, suffix = buffer.split('_')
-        else:
-            prefix, suffix = prefix.strip('_'), buffer
-
-        assert len(prefix) == ABID_PREFIX_LEN - 1   # length without trailing _
-        assert len(suffix) == ABID_SUFFIX_LEN, f'Suffix {suffix} from {buffer} was not {ABID_SUFFIX_LEN} chars long'
-
-        return cls(
-            prefix=abid_part_from_prefix(prefix),
-            ts=suffix[0:10].upper(),
-            uri=suffix[10:18].upper(),
-            subtype=suffix[18:20].upper(),
-            rand=suffix[20:26].upper(),
-        )
-    
-    @property
-    def uri_salt(self) -> str:
-        return DEFAULT_ABID_URI_SALT
-
-    @property
-    def suffix(self):
-        return ''.join((self.ts, self.uri, self.subtype, self.rand))
-    
-    @property
-    def ulid(self) -> ulid.ULID:
-        return ulid.parse(self.suffix)
-
-    @property
-    def uuid(self) -> UUID:
-        return self.ulid.uuid
-
-    @property
-    def uuid6(self) -> uuid6.UUID:
-        return uuid6.UUID(hex=self.uuid.hex)
-
-    @property
-    def typeid(self) -> TypeID:
-        return TypeID.from_uuid(prefix=self.prefix.strip('_'), suffix=self.uuid6)
-
-    @property
-    def datetime(self) -> datetime:
-        return self.ulid.timestamp().datetime
-
-
-
-####################################################
-
-
-@enforce_types
-def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
-    """
-    https://example.com -> 'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25' (example.com)
-    """
-    if isinstance(uri, bytes):
-        uri_str: str = uri.decode()
-    else:
-        uri_str = str(uri)
-
-    # only hash the domain part of URLs
-    if '://' in uri_str:
-        try:
-            domain = urlparse(uri_str).netloc
-            if domain:
-                uri_str = domain
-        except AttributeError:
-            pass
-    
-    # the uri hash is the sha256 of the domain + salt
-    uri_bytes = uri_str.encode('utf-8') + salt.encode('utf-8')
-
-    return hashlib.sha256(uri_bytes).hexdigest().upper()
-
-@enforce_types
-def abid_part_from_prefix(prefix: str) -> str:
-    """
-    'snp_'
-    """
-    # if prefix is None:
-    #     return 'obj_'
-
-    prefix = prefix.strip('_').lower()
-    assert len(prefix) == 3
-    return prefix + '_'
-
-@enforce_types
-def abid_part_from_uri(uri: Any, salt: str=DEFAULT_ABID_URI_SALT) -> str:
-    """
-    'E4A5CCD9'     # takes first 8 characters of sha256(url)
-    """
-    uri = str(uri).strip()
-    assert uri not in ('None', '')
-    return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
-
-@enforce_types
-def abid_part_from_ts(ts: datetime) -> str:
-    """
-    '01HX9FPYTR'   # produces 10 character Timestamp section of ulid based on added date
-    """
-    return str(ulid.from_timestamp(ts))[:ABID_TS_LEN]
-
-@enforce_types
-def ts_from_abid(abid: str) -> datetime:
-    return ulid.parse(abid.split('_', 1)[-1]).timestamp().datetime
-
-@enforce_types
-def abid_part_from_subtype(subtype: str | int) -> str:
-    """
-    Snapshots have 01 type, other objects have other subtypes like wget/media/etc.
-    Also allows us to change the ulid spec later by putting special sigil values here.
-    """
-    subtype = str(subtype)
-    if len(subtype) == ABID_SUBTYPE_LEN:
-        return subtype
-
-    return hashlib.sha256(subtype.encode('utf-8')).hexdigest()[:ABID_SUBTYPE_LEN].upper()
-
-@enforce_types
-def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
-    """
-    'ZYEBQE'   # takes last 6 characters of randomness from existing legacy uuid db field
-    """
-    if rand is None:
-        # if it's None we generate a new random 6 character hex string
-        return str(ulid.new())[-ABID_RAND_LEN:]
-    elif isinstance(rand, UUID):
-        # if it's a uuid we take the last 6 characters of the ULID represation of it
-        return str(ulid.from_uuid(rand))[-ABID_RAND_LEN:]
-    elif isinstance(rand, int):
-        # if it's a BigAutoInteger field we convert it from an int to a 0-padded string
-        rand_str = str(rand)[-ABID_RAND_LEN:]
-        padding_needed = ABID_RAND_LEN - len(rand_str)
-        rand_str = ('0'*padding_needed) + rand_str
-        return rand_str
-
-    # otherwise treat it as a string, take the last 6 characters of it verbatim
-    return str(rand)[-ABID_RAND_LEN:].upper()
-
-
-@enforce_types
-def abid_hashes_from_values(prefix: str, ts: datetime, uri: Any, subtype: str | int, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> Dict[str, str]:
-    return {
-        'prefix': abid_part_from_prefix(prefix),
-        'ts': abid_part_from_ts(ts),
-        'uri': abid_part_from_uri(uri, salt=salt),
-        'subtype': abid_part_from_subtype(subtype),
-        'rand': abid_part_from_rand(rand),
-        # 'salt': don't add this, salt combined with uri above to form a single hash
-    }
-
-@enforce_types
-def abid_from_values(prefix: str, ts: datetime, uri: str, subtype: str, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> ABID:
-    """
-    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
-    """
-
-    abid = ABID(**abid_hashes_from_values(prefix, ts, uri, subtype, rand, salt=salt))
-    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for ts={ts} uri={uri} subtyp={subtype} rand={rand}'
-    return abid
@@ -1,174 +0,0 @@
-__package__ = 'archivebox.abid_utils'
-
-from typing import Any
-
-from django.contrib import admin, messages
-from django.core.exceptions import ValidationError
-from django.utils.html import format_html
-from django.utils.safestring import mark_safe
-from django.shortcuts import redirect
-
-from django_object_actions import DjangoObjectActions, action
-
-from archivebox.misc.util import parse_date
-
-from .abid import ABID
-
-
-def highlight_diff(display_val: Any, compare_val: Any, invert: bool=False, color_same: str | None=None, color_diff: str | None=None):
-    """highlight each character in red that differs with the char at the same index in compare_val"""
-
-    display_val = str(display_val)
-    compare_val = str(compare_val)
-
-    if len(compare_val) < len(display_val):
-        compare_val += ' ' * (len(display_val) - len(compare_val))
-
-    similar_color, highlighted_color = color_same or 'inherit', color_diff or 'red'
-    if invert:
-        similar_color, highlighted_color = color_same or 'green', color_diff or 'inherit'
-
-    return mark_safe(''.join(
-        format_html('<span style="color: {};">{}</span>', highlighted_color, display_val[i])
-        if display_val[i] != compare_val[i] else
-        format_html('<span style="color: {};">{}</span>', similar_color, display_val[i])
-        for i in range(len(display_val))
-    ))
-
-def get_abid_info(self, obj, request=None):
-    from archivebox.api.auth import get_or_create_api_token
-    
-    try:
-        #abid_diff = f' != obj.ABID: {highlight_diff(obj.ABID, obj.abid)} ❌' if str(obj.ABID) != str(obj.abid) else ' == .ABID ✅'
-
-        fresh_values = obj.ABID_FRESH_VALUES
-        fresh_hashes = obj.ABID_FRESH_HASHES
-        fresh_diffs = obj.ABID_FRESH_DIFFS
-        fresh_abid = ABID(**fresh_hashes)
-        
-        fresh_abid_diff = f'❌ != &nbsp; .fresh_abid: {highlight_diff(fresh_abid, obj.ABID)}' if str(fresh_abid) != str(obj.ABID) else '✅'
-        fresh_uuid_diff = f'❌ != &nbsp; .fresh_uuid: {highlight_diff(fresh_abid.uuid, obj.ABID.uuid)}' if str(fresh_abid.uuid) != str(obj.ABID.uuid) else '✅'
-
-        id_pk_diff = f'❌ !=  .pk: {highlight_diff(obj.pk, obj.id)}' if str(obj.pk) != str(obj.id) else '✅'
-
-        fresh_ts = parse_date(fresh_values['ts']) or None
-        ts_diff = f'❌ != {highlight_diff( fresh_hashes["ts"], obj.ABID.ts)}' if  fresh_hashes["ts"] != obj.ABID.ts else '✅'
-
-        derived_uri = fresh_hashes['uri']
-        uri_diff = f'❌ != {highlight_diff(derived_uri, obj.ABID.uri)}' if derived_uri != obj.ABID.uri else '✅'
-
-        derived_subtype = fresh_hashes['subtype']
-        subtype_diff = f'❌ != {highlight_diff(derived_subtype, obj.ABID.subtype)}' if derived_subtype != obj.ABID.subtype else '✅'
-
-        derived_rand = fresh_hashes['rand']
-        rand_diff = f'❌ != {highlight_diff(derived_rand, obj.ABID.rand)}' if derived_rand != obj.ABID.rand else '✅'
-
-        return format_html(
-            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
-            '''
-            <a href="{}" style="font-size: 16px; font-family: monospace; user-select: all; border-radius: 8px; background-color: #ddf; padding: 3px 5px; border: 1px solid #aaa; margin-bottom: 8px; display: inline-block; vertical-align: top;">{}</a> &nbsp; &nbsp; <a href="{}" style="color: limegreen; font-size: 0.9em; vertical-align: 1px; font-family: monospace;">📖 API DOCS</a>
-            <br/><hr/>
-            <div style="opacity: 0.8">
-            &nbsp; &nbsp; <small style="opacity: 0.8">.id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; {}</small><br/>
-            &nbsp; &nbsp; <small style="opacity: 0.8">.abid.uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; {}</small><br/>
-            &nbsp; &nbsp; <small style="opacity: 0.8">.abid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</small><br/>
-            <hr/>
-            &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; {}</code> &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
-            &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; {}</code> &nbsp;&nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px;"><b>{}</b></code> <span style="display:inline-block; vertical-align: -4px; width: 330px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}: <code style="user-select: all">{}</code></span><br/>
-            &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
-            &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code></code>
-            <br/><hr/>
-            <span style="color: #f375a0">{}</span> <code style="color: red"><b>{}</b></code> {}
-            </div>
-            ''',
-            obj.api_url + (f'?api_key={get_or_create_api_token(request.user)}' if request and request.user else ''), obj.api_url, obj.api_docs_url,
-            highlight_diff(obj.id, obj.ABID.uuid, invert=True), mark_safe(id_pk_diff),
-            highlight_diff(obj.ABID.uuid, obj.id, invert=True), mark_safe(fresh_uuid_diff),
-            highlight_diff(obj.abid, fresh_abid), mark_safe(fresh_abid_diff),
-            # str(fresh_abid.uuid), mark_safe(fresh_uuid_diff),
-            # str(fresh_abid), mark_safe(fresh_abid_diff),
-            highlight_diff(obj.ABID.ts,  fresh_hashes['ts']), highlight_diff(str(obj.ABID.uuid)[0:14], str(fresh_abid.uuid)[0:14]), mark_safe(ts_diff), obj.abid_ts_src, fresh_ts and fresh_ts.isoformat(),
-            highlight_diff(obj.ABID.uri, derived_uri), highlight_diff(str(obj.ABID.uuid)[14:26], str(fresh_abid.uuid)[14:26]), mark_safe(uri_diff), obj.abid_uri_src, str(fresh_values['uri']),
-            highlight_diff(obj.ABID.subtype, derived_subtype), highlight_diff(str(obj.ABID.uuid)[26:28], str(fresh_abid.uuid)[26:28]), mark_safe(subtype_diff), obj.abid_subtype_src, str(fresh_values['subtype']),
-            highlight_diff(obj.ABID.rand, derived_rand), highlight_diff(str(obj.ABID.uuid)[28:36], str(fresh_abid.uuid)[28:36]), mark_safe(rand_diff), obj.abid_rand_src, str(fresh_values['rand'])[-7:],
-            'Some values the ABID depends on have changed since the ABID was issued:' if fresh_diffs else '',
-            ", ".join(diff['abid_src'] for diff in fresh_diffs.values()),
-            '(clicking "Regenerate ABID" in the upper right will assign a new ABID, breaking any external references to the old ABID)' if fresh_diffs else '',
-        )
-    except Exception as e:
-        # import ipdb; ipdb.set_trace()
-        return str(e)
-
-
-class ABIDModelAdmin(DjangoObjectActions, admin.ModelAdmin):
-    list_display = ('created_at', 'created_by', 'abid')
-    sort_fields = ('created_at', 'created_by', 'abid')
-    readonly_fields = ('created_at', 'modified_at', 'abid_info')
-    # fields = [*readonly_fields]
-    
-    change_actions = ("regenerate_abid",)
-    # changelist_actions = ("regenerate_abid",)
-
-    def _get_obj_does_not_exist_redirect(self, request, opts, object_id):
-        try:
-            object_pk = self.model.id_from_abid(object_id)
-            return redirect(self.request.path.replace(object_id, object_pk), permanent=False)
-        except (self.model.DoesNotExist, ValidationError):
-            pass
-        return super()._get_obj_does_not_exist_redirect(request, opts, object_id)       # type: ignore
-    
-    def queryset(self, request):
-        self.request = request
-        return super().queryset(request)                                                # type: ignore
-    
-    def change_view(self, request, object_id, form_url="", extra_context=None):
-        self.request = request
-        return super().change_view(request, object_id, form_url, extra_context)
-
-    def get_form(self, request, obj=None, **kwargs):
-        self.request = request
-        form = super().get_form(request, obj, **kwargs)
-        if 'created_by' in form.base_fields:
-            form.base_fields['created_by'].initial = request.user
-            
-        if obj:
-            if obj.ABID_FRESH_DIFFS:
-                messages.warning(request, "The ABID is not in sync with the object! See the API Identifiers section below for more info...")
-
-        return form
-
-    def get_formset(self, request, formset=None, obj=None, **kwargs):
-        formset = super().get_formset(request, formset, obj, **kwargs)                  # type: ignore
-        formset.form.base_fields['created_at'].disabled = True
-        
-        return formset
-
-    def save_model(self, request, obj, form, change):
-        self.request = request
-
-        old_abid = getattr(obj, '_previous_abid', None) or obj.abid
-
-        super().save_model(request, obj, form, change)
-        obj.refresh_from_db()
-
-        new_abid = obj.abid
-        if new_abid != old_abid:
-            messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
-        # import ipdb; ipdb.set_trace()
-
-    @admin.display(description='API Identifiers')
-    def abid_info(self, obj):
-        return get_abid_info(self, obj, request=self.request)
-
-    @action(label="Regenerate ABID", description="Re-Generate the ABID based on fresh values")
-    def regenerate_abid(self, request, obj):
-        old_abid = str(obj.abid)
-        obj.abid = obj.issue_new_abid(overwrite=True)
-        obj.save()
-        obj.refresh_from_db()
-        new_abid = str(obj.abid)
-
-        if new_abid != old_abid:
-            messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
-        else:
-            messages.success(request, "The ABID was not regenerated, it is already up-to-date with the object.")
@@ -1,7 +0,0 @@
-from django.apps import AppConfig
-
-
-class AbidUtilsConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    
-    name = 'abid_utils'
@@ -1,509 +0,0 @@
-"""
-This file provides the Django ABIDField and ABIDModel base model to inherit from.
-"""
-
-
-from typing import Any, Dict, Union, List, Set, cast
-
-from uuid import uuid4
-from functools import partial
-from charidfield import CharIDField  # type: ignore[import-untyped]
-
-from django.contrib import admin
-from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
-from django.db import models
-from django.utils import timezone
-from django.db.utils import OperationalError
-from django.contrib.auth import get_user_model
-from django.urls import reverse_lazy
-
-from django_stubs_ext.db.models import TypedModelMeta
-
-from .abid import (
-    ABID,
-    ABID_LEN,
-    ABID_RAND_LEN,
-    ABID_SUFFIX_LEN,
-    DEFAULT_ABID_PREFIX,
-    DEFAULT_ABID_URI_SALT,
-    abid_part_from_prefix,
-    abid_hashes_from_values,
-    ts_from_abid,
-    abid_part_from_ts,
-)
-
-####################################################
-
-
-# Database Field for typeid/ulid style IDs with a prefix, e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ
-ABIDField = partial(
-    CharIDField,
-    max_length=ABID_LEN,
-    help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
-    default=None,
-    null=True,
-    blank=True,
-    db_index=True,
-    unique=True,
-)
-
-def get_or_create_system_user_pk(username='system'):
-    """Get or create a system user with is_superuser=True to be the default owner for new DB rows"""
-
-    User = get_user_model()
-
-    # if only one user exists total, return that user
-    if User.objects.filter(is_superuser=True).count() == 1:
-        return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
-
-    # otherwise, create a dedicated "system" user
-    user, _was_created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
-    return user.pk
-
-
-class AutoDateTimeField(models.DateTimeField):
-    # def pre_save(self, model_instance, add):
-    #     return timezone.now()
-    pass
-
-class ABIDError(Exception):
-    pass
-
-
-class ABIDModel(models.Model):
-    """
-    Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
-    """
-    abid_prefix: str = DEFAULT_ABID_PREFIX            # e.g. 'tag_'
-    abid_ts_src = 'self.created_at'                  # e.g. 'self.created_at'
-    abid_uri_src = 'None'                            # e.g. 'self.uri'                (MUST BE SET)
-    abid_subtype_src = 'self.__class__.__name__'     # e.g. 'self.extractor'
-    abid_rand_src = 'self.id'                        # e.g. 'self.uuid' or 'self.id'
-    abid_salt: str = DEFAULT_ABID_URI_SALT           # combined with self.uri to anonymize hashes on a per-install basis (default is shared globally with all users, means everyone will hash ABC to -> 123 the same around the world, makes it easy to share ABIDs across installs and see if they are for the same URI. Change this if you dont want your hashes to be guessable / in the same hash space as all other users)
-    abid_drift_allowed: bool = False                 # set to True to allow abid_field values to change after a fixed ABID has been issued (NOT RECOMMENDED: means values can drift out of sync from original ABID)
-
-    # id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
-    # abid = ABIDField(prefix=abid_prefix)
-
-    # created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    # created_at = AutoDateTimeField(default=None, null=False, db_index=True)
-    # modified_at = models.DateTimeField(auto_now=True)
-
-    _prefetched_objects_cache: Dict[str, Any]
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    @admin.display(description='Summary')
-    def __str__(self) -> str:
-        return f'[{self.abid or (self.abid_prefix + "NEW")}] {self.__class__.__name__} {eval(self.abid_uri_src)}'
-
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        """Overriden __init__ method ensures we have a stable creation timestamp that fields can use within initialization code pre-saving to DB."""
-        super().__init__(*args, **kwargs)
-        # pre-compute a stable timestamp of the obj init time (with abid.ts precision limit applied) for use when object is first created,
-        # some other fields depend on a timestamp at creation time, and it's nice to have one common timestamp they can all share.
-        # Used as an alternative to auto_now_add=True + auto_now=True which can produce two different times & requires saving to DB to get the TS.
-        # (ordinarily fields cant depend on other fields until the obj is saved to db and recalled)
-        self._init_timestamp = ts_from_abid(abid_part_from_ts(timezone.now()))
-
-    def clean(self, abid_drift_allowed: bool | None=None) -> None:
-        if self._state.adding:
-            # only runs once when a new object is first saved to the DB
-            # sets self.id, self.pk, self.created_by, self.created_at, self.modified_at
-            self._previous_abid = None
-            self.abid = str(self.issue_new_abid())
-
-        else:
-            # otherwise if updating, make sure none of the field changes would invalidate existing ABID
-            abid_diffs = self.ABID_FRESH_DIFFS
-            if abid_diffs:
-                # change has invalidated the existing ABID, raise a nice ValidationError pointing out which fields caused the issue
-
-                keys_changed = ', '.join(diff['abid_src'] for diff in abid_diffs.values())
-                full_summary = (
-                    f"This {self.__class__.__name__}(abid={str(self.ABID)}) was assigned a fixed, unique ID (ABID) based on its contents when it was created. " +
-                    f"\nYou must reduce your changes to not affect these fields [{keys_changed}], or create a new {self.__class__.__name__} object instead."
-                )
-
-                change_error = ValidationError({
-                    **{
-                        # url: ValidationError('Cannot update self.url= https://example.com/old -> https://example.com/new ...')
-                        diff['abid_src'].replace('self.', '')
-                            if (diff['old_val'] != diff['new_val']) and hasattr(self, diff['abid_src'].replace('self.', ''))
-                            else NON_FIELD_ERRORS
-                        : ValidationError(
-                            'Cannot update %(abid_src)s= "%(old_val)s" -> "%(new_val)s" (would alter %(model)s.ABID.%(key)s=%(old_hash)s to %(new_hash)s)',
-                            code='ABIDConflict',
-                            params=diff,
-                        )
-                        for diff in abid_diffs.values()
-                    },
-                    NON_FIELD_ERRORS: ValidationError(full_summary),
-                })
-
-                allowed_to_invalidate_abid = self.abid_drift_allowed if (abid_drift_allowed is None) else abid_drift_allowed
-                if allowed_to_invalidate_abid:
-                    # print(f'\n#### WARNING: Change allowed despite it invalidating the ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})!', self.abid)
-                    # print(change_error)
-                    # print('--------------------------------------------------------------------------------------------------')
-                    pass
-                else:
-                    print(f'\n#### ERROR:   Change blocked because it would invalidate ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})', self.abid)
-                    print(change_error)
-                    print('--------------------------------------------------------------------------------------------------')
-                    raise change_error
-
-    def save(self, *args: Any, abid_drift_allowed: bool | None=None, **kwargs: Any) -> None:
-        """Overriden save method ensures new ABID is generated while a new object is first saving."""
-
-        self.clean(abid_drift_allowed=abid_drift_allowed)
-
-        return super().save(*args, **kwargs)
-    
-    @classmethod
-    def id_from_abid(cls, abid: str) -> str:
-        return str(cls.objects.only('pk').get(abid=cls.abid_prefix + str(abid).split('_', 1)[-1]).pk)
-
-    @property
-    def ABID_SOURCES(self) -> Dict[str, str]:
-        """"Get the dict of fresh ABID component values based on the live object's properties."""
-        assert self.abid_prefix
-        return {
-            'prefix': 'self.abid_prefix',             # defined as static class vars at build time
-            'ts': self.abid_ts_src,
-            'uri': self.abid_uri_src,
-            'subtype': self.abid_subtype_src,
-            'rand': self.abid_rand_src,
-            'salt': 'self.abid_salt',                 # defined as static class vars at build time
-        }
-
-    @property
-    def ABID_FRESH_VALUES(self) -> Dict[str, Any]:
-        """"Get the dict of fresh ABID component values based on the live object's properties."""
-        abid_sources = self.ABID_SOURCES
-        assert all(src != 'None' for src in abid_sources.values())
-        return {
-            'prefix': eval(abid_sources['prefix']),
-            'ts': eval(abid_sources['ts']),
-            'uri': eval(abid_sources['uri']),
-            'subtype': eval(abid_sources['subtype']),
-            'rand': eval(abid_sources['rand']),
-            'salt': eval(abid_sources['salt']),
-        }
-    
-    @property
-    def ABID_FRESH_HASHES(self) -> Dict[str, str]:
-        """"Get the dict of fresh ABID component hashes based on the live object's properties."""
-        abid_values = self.ABID_FRESH_VALUES
-        assert all(val for val in abid_values.values())
-        return abid_hashes_from_values(
-            prefix=abid_values['prefix'],
-            ts=abid_values['ts'],
-            uri=abid_values['uri'],
-            subtype=abid_values['subtype'],
-            rand=abid_values['rand'],
-            salt=abid_values['salt'],
-        )
-    
-    @property
-    def ABID_FRESH_DIFFS(self) -> Dict[str, Dict[str, Any]]:
-        """Get the dict of discrepancies between the existing saved ABID and a new fresh ABID computed based on the live object."""
-        existing_abid = self.ABID
-        existing_values = {} if self._state.adding else self.__class__.objects.get(pk=self.pk).ABID_FRESH_VALUES
-        abid_sources = self.ABID_SOURCES
-        fresh_values = self.ABID_FRESH_VALUES
-        fresh_hashes = self.ABID_FRESH_HASHES
-        return {
-            key: {
-                'key': key,
-                'model': self.__class__.__name__,
-                'pk': self.pk,
-                'abid_src': abid_sources[key],
-                'old_val': existing_values.get(key, None),
-                'old_hash': getattr(existing_abid, key),
-                'new_val': fresh_values[key],
-                'new_hash': new_hash,
-                'summary': f'{abid_sources[key]}= "{existing_values.get(key, None)}" -> "{fresh_values[key]}" (would alter {self.__class__.__name__.lower()}.ABID.{key}={getattr(existing_abid, key)} to {new_hash})',
-            }
-            for key, new_hash in fresh_hashes.items()
-            if getattr(existing_abid, key) != new_hash
-        }
-
-    def issue_new_abid(self, overwrite=False) -> ABID:
-        """
-        Issue a new ABID based on the current object's properties, can only be called once on new objects (before they are saved to DB).
-        """
-        if not overwrite:
-            assert self._state.adding, 'Can only issue new ABID when model._state.adding is True'
-        assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})'
-
-        # Setup Field defaults to be ready for ABID generation
-        self.abid = None
-        self.id = self.id or uuid4()
-        self.pk = self.id
-        self.created_at = self.created_at or self._init_timestamp  # cut off precision to match precision of TS component
-        self.modified_at = self.modified_at or self.created_at
-        self.created_by_id = (hasattr(self, 'created_by_id') and self.created_by_id) or get_or_create_system_user_pk()
-        
-        # Compute fresh ABID values & hashes based on object's live properties
-        abid_fresh_values = self.ABID_FRESH_VALUES
-        assert all(abid_fresh_values.values()), f'All ABID_FRESH_VALUES must be set {abid_fresh_values}'
-        abid_fresh_hashes = self.ABID_FRESH_HASHES
-        assert all(abid_fresh_hashes.values()), f'All ABID_FRESH_HASHES must be able to be generated {abid_fresh_hashes}'
-        
-        new_abid = ABID(**abid_fresh_hashes)
-        
-        assert new_abid.ulid and new_abid.uuid and new_abid.typeid, f'Failed to calculate {abid_fresh_values["prefix"]}_ABID for {self.__class__.__name__}'
-
-        return new_abid
-
-    @property
-    def ABID(self) -> ABID:
-        """
-        Get the object's existing ABID (from self.abid if it's already saved to DB, otherwise generated fresh)
-        e.g. -> ABID(ts='01HX9FPYTR', uri='E4A5CCD9', subtype='00', rand='ZYEBQE')
-        """
-
-        if self.abid:
-            return ABID.parse(cast(str, self.abid))
-        
-        return self.issue_new_abid()
-
-    # These are all example helpers to make it easy to access alternate formats of the ABID.*, only add them if you actually need them
-    # @property
-    # def UUID(self) -> UUID:
-    #     """
-    #     Get a uuid.UUID (v4) representation of the object's ABID.
-    #     """
-    #     return self.ABID.uuid
-    
-    # @property
-    # def uuid(self) -> str:
-    #     """
-    #     Get a str uuid.UUID (v4) representation of the object's ABID.
-    #     """
-    #     return str(self.ABID.uuid)
-    
-    # @property
-    # def ULID(self) -> ULID:
-    #     """
-    #     Get a ulid.ULID representation of the object's ABID.
-    #     """
-    #     return self.ABID.ulid
-
-    # @property
-    # def TypeID(self) -> TypeID:
-    #     """
-    #     Get a typeid.TypeID (stripe-style) representation of the object's ABID.
-    #     """
-    #     return self.ABID.typeid
-    
-    @property
-    def api_url(self) -> str:
-        """
-        Compute the REST API URL to access this object.
-        e.g. /api/v1/core/snapshot/snp_01BJQMF54D093DXEAWZ6JYRP
-        """
-        return reverse_lazy('api-1:get_any', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
-
-    @property
-    def api_docs_url(self) -> str:
-        """
-        Compute the REST API Documentation URL to learn about accessing this object.
-        e.g. /api/v1/docs#/Core%20Models/api_v1_core_get_snapshots
-        """
-        return f'/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}'
-
-    @property
-    def admin_change_url(self) -> str:
-        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
-
-    def get_absolute_url(self):
-        return self.api_docs_url
-
-
-
-class ModelWithHealthStats(models.Model):
-    num_uses_failed = models.PositiveIntegerField(default=0)
-    num_uses_succeeded = models.PositiveIntegerField(default=0)
-    
-    class Meta:
-        abstract = True
-    
-    def record_health_failure(self) -> None:
-        self.num_uses_failed += 1
-        self.save()
-
-    def record_health_success(self) -> None:
-        self.num_uses_succeeded += 1
-        self.save()
-        
-    def reset_health(self) -> None:
-        # move all the failures to successes when resetting so we dont lose track of the total count
-        self.num_uses_succeeded = self.num_uses_failed + self.num_uses_succeeded
-        self.num_uses_failed = 0
-        self.save()
-        
-    @property
-    def health(self) -> int:
-        total_uses = max((self.num_uses_failed + self.num_uses_succeeded, 1))
-        success_pct = (self.num_uses_succeeded / total_uses) * 100
-        return round(success_pct)
-
-
-
-
-
-
-
-
-
-
-####################################################
-
-# Django helpers
-def find_all_abid_prefixes() -> Dict[str, type[models.Model]]:
-    """
-    Return the mapping of all ABID prefixes to their models.
-    e.g. {'tag_': core.models.Tag, 'snp_': core.models.Snapshot, ...}
-    """
-    import django.apps
-    prefix_map = {}
-
-    for model in django.apps.apps.get_models():
-        abid_prefix = getattr(model, 'abid_prefix', None)
-        if abid_prefix:
-            prefix_map[abid_prefix] = model
-    return prefix_map
-
-def find_prefix_for_abid(abid: ABID) -> str:
-    """
-    Find the correct prefix for a given ABID that may have be missing a prefix (slow).
-    e.g. ABID('obj_01BJQMF54D093DXEAWZ6JYRPAQ') -> 'snp_'
-    """
-    # if existing abid prefix is correct, lookup is easy
-    model = find_model_from_abid(abid)
-    if model:
-        assert issubclass(model, ABIDModel)
-        return model.abid_prefix
-
-    # prefix might be obj_ or missing, fuzzy-search to find any object that matches
-    return find_obj_from_abid_rand(abid)[0].abid_prefix
-
-def find_model_from_abid_prefix(prefix: str) -> type[ABIDModel] | None:
-    """
-    Return the Django Model that corresponds to a given ABID prefix.
-    e.g. 'tag_' -> core.models.Tag
-    """
-    prefix = abid_part_from_prefix(prefix)
-
-    import django.apps
-
-    for model in django.apps.apps.get_models():
-        if not issubclass(model, ABIDModel): continue   # skip non-ABID-enabled models
-        if not hasattr(model, 'objects'): continue      # skip abstract models
-
-        if (model.abid_prefix == prefix):
-            return model
-
-    return None
-
-def find_model_from_abid(abid: ABID) -> type[models.Model] | None:
-    """
-    Shortcut for find_model_from_abid_prefix(abid.prefix)
-    """
-    return find_model_from_abid_prefix(abid.prefix)
-
-def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDModel]:
-    """
-    Find an object corresponding to an ABID by exhaustively searching using its random suffix (slow).
-    e.g. 'obj_....................JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
-    Honestly should only be used for debugging, no reason to expose this ability to users.
-    """
-
-    # convert str to ABID if necessary
-    if isinstance(rand, ABID):
-        abid: ABID = rand
-    else:
-        rand = str(rand)
-        if len(rand) < ABID_SUFFIX_LEN:
-            padding_needed = ABID_SUFFIX_LEN - len(rand)
-            rand = ('0'*padding_needed) + rand
-        abid = ABID.parse(rand)
-
-    import django.apps
-
-    partial_matches: List[ABIDModel] = []
-
-    models_to_try = cast(Set[type[models.Model]], set(filter(bool, (
-        model,
-        find_model_from_abid(abid),
-        *django.apps.apps.get_models(),
-    ))))
-    # print(abid, abid.rand, abid.uuid, models_to_try)
-
-    for model in models_to_try:
-        if not issubclass(model, ABIDModel): continue   # skip Models that arent ABID-enabled
-        if not hasattr(model, 'objects'): continue      # skip abstract Models
-        assert hasattr(model, 'objects')                # force-fix for type hint nit about missing manager https://github.com/typeddjango/django-stubs/issues/1684
-
-        # continue on to try fuzzy searching by randomness portion derived from uuid field
-        try:
-            qs = []
-            if hasattr(model, 'abid'):
-                qs = model.objects.filter(abid__endswith=abid.rand)
-            elif hasattr(model, 'uuid'):
-                qs = model.objects.filter(uuid__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
-            elif hasattr(model, 'id'):
-                # NOTE: this only works on SQLite where every column is a string
-                # other DB backends like postgres dont let you do __endswith if this is a BigAutoInteger field
-                
-                # try to search for uuid=...-2354352
-                # try to search for id=...2354352
-                # try to search for id=2354352
-                qs = model.objects.filter(
-                    models.Q(id__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
-                    | models.Q(id__endswith=abid.rand)
-                    | models.Q(id__startswith=str(int(abid.rand)) if abid.rand.isdigit() else abid.rand)
-                )
-
-            for obj in qs:
-                if abid in (str(obj.ABID), str(obj.id), str(obj.pk), str(obj.abid)):
-                    # found exact match, no need to keep iterating
-                    return [obj]
-                partial_matches.append(obj)
-        except OperationalError as err:
-            print(f'[!] WARNING: Got error while trying to iterate through QuerySet for {model}:', err, '\n')
-
-    return partial_matches
-
-def find_obj_from_abid(abid: ABID, model=None, fuzzy=False) -> Any:
-    """
-    Find an object with a given ABID by filtering possible models for a matching abid/uuid/id (fast).
-    e.g. 'snp_01BJQMF54D093DXEAWZ6JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
-    """
-
-    model = model or find_model_from_abid(abid)
-    assert model, f'Could not find model that could match this ABID type: {abid}'
-
-    try:
-        if hasattr(model, 'abid'):
-            return model.objects.get(abid__endswith=abid.suffix)
-        if hasattr(model, 'uuid'):
-            return model.objects.get(uuid=abid.uuid)
-        return model.objects.get(id=abid.uuid)
-    except model.DoesNotExist:
-        # if the model has an abid field then it shouldve matched, pointless to fuzzy search in that case
-        if hasattr(model, 'abid') or (not fuzzy):
-            raise
-
-    # continue on to try fuzzy searching by randomness portion derived from uuid field
-    match_by_rand = find_obj_from_abid_rand(abid, model=model)
-    if match_by_rand:
-        if match_by_rand[0].abid_prefix != abid.prefix:
-            print(f'[!] WARNING: fetched object {match_by_rand} even though prefix {abid.prefix} doesnt match!', abid, '\n')
-        return match_by_rand
-
-    raise model.DoesNotExist
-
@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.
@@ -1,571 +0,0 @@
-__package__ = 'archivebox.actors'
-
-import os
-import time
-from typing import ClassVar, Generic, TypeVar, Any, Literal, Type, Iterable, cast, get_args
-from datetime import timedelta
-from multiprocessing import Process, cpu_count
-from threading import Thread, get_native_id
-
-import psutil
-from rich import print
-from statemachine import State, StateMachine, registry
-from statemachine.mixins import MachineMixin
-
-from django import db
-from django.db.models import QuerySet, sql, Q
-from django.db.models import Model as DjangoModel
-from django.utils import timezone
-from django.utils.functional import classproperty
-
-from .models import ModelWithStateMachine
-
-# from archivebox.logging_util import TimedProgress
-
-class ActorObjectAlreadyClaimed(Exception):
-    """Raised when the Actor tries to claim the next object from the queue but it's already been claimed by another Actor"""
-    pass
-
-class ActorQueueIsEmpty(Exception):
-    """Raised when the Actor tries to get the next object from the queue but it's empty"""
-    pass
-
-CPU_COUNT = cpu_count()
-DEFAULT_MAX_TICK_TIME = 60
-DEFAULT_MAX_CONCURRENT_ACTORS = min(max(2, int(CPU_COUNT * 0.6)), 8)   # 2 < 60% * num available cpu cores < 8
-
-limit = lambda n, max: min(n, max)
-
-LaunchKwargs = dict[str, Any]
-ObjectState = State | str
-ObjectStateList = Iterable[ObjectState]
-
-ModelType = TypeVar('ModelType', bound=ModelWithStateMachine)
-
-class ActorType(Generic[ModelType]):
-    """
-    Base class for all actors. Usage:
-    
-    class FaviconActor(ActorType[FaviconArchiveResult]):
-        FINAL_STATES: ClassVar[tuple[str, ...]] = ('succeeded', 'failed', 'skipped')
-        ACTIVE_STATE: ClassVar[str] = 'started'
-        
-        @classmethod
-        def qs(cls) -> QuerySet[FaviconArchiveResult]:
-            return ArchiveResult.objects.filter(extractor='favicon')   # or leave the default: FaviconArchiveResult.objects.all()
-    """
-    
-    ### Class attributes (defined on the class at compile-time when ActorType[MyModel] is defined)
-    Model: Type[ModelType]
-    StateMachineClass: Type[StateMachine]
-    
-    STATE_FIELD_NAME: ClassVar[str]
-    ACTIVE_STATE: ClassVar[ObjectState]
-    FINAL_STATES: ClassVar[ObjectStateList]
-    EVENT_NAME: ClassVar[str] = 'tick'                                    # the event name to trigger on the obj.sm: StateMachine (usually 'tick')
-    
-    CLAIM_ORDER: ClassVar[tuple[str, ...]] = ('retry_at',)                # the .order(*args) to claim the queue objects in, use ('?',) for random order
-    CLAIM_FROM_TOP_N: ClassVar[int] = CPU_COUNT * 10                      # the number of objects to consider when atomically getting the next object from the queue
-    CLAIM_ATOMIC: ClassVar[bool] = True                                   # whether to atomically fetch+claim the next object in one query, or fetch and lock it in two queries
-    
-    MAX_TICK_TIME: ClassVar[int] = DEFAULT_MAX_TICK_TIME                  # maximum duration in seconds to process a single object
-    MAX_CONCURRENT_ACTORS: ClassVar[int] = DEFAULT_MAX_CONCURRENT_ACTORS  # maximum number of concurrent actors that can be running at once
-    
-    _SPAWNED_ACTOR_PIDS: ClassVar[list[psutil.Process]] = []      # used to record all the pids of Actors spawned on the class
-    
-    ### Instance attributes (only used within an actor instance inside a spawned actor thread/process)
-    pid: int
-    idle_count: int = 0
-    launch_kwargs: LaunchKwargs = {}
-    mode: Literal['thread', 'process'] = 'process'
-    
-    def __init_subclass__(cls) -> None:
-        """
-        Executed at class definition time (i.e. during import of any file containing class MyActor(ActorType[MyModel]): ...).
-        Loads the django Model from the Generic[ModelType] TypeVar arg and populates any missing class-level config using it.
-        """
-        if getattr(cls, 'Model', None) is None:
-            cls.Model = cls._get_model_from_generic_typevar()
-        cls._populate_missing_classvars_from_model(cls.Model)
-    
-    def __init__(self, mode: Literal['thread', 'process']|None=None, **launch_kwargs: LaunchKwargs):
-        """
-        Executed right before the Actor is spawned to create a unique Actor instance for that thread/process.
-        actor_instance.runloop() is then executed from inside the newly spawned thread/process.
-        """
-        self.mode = mode or self.mode
-        self.launch_kwargs = launch_kwargs or dict(self.launch_kwargs)
-    
-
-    ### Private Helper Methods: Not desiged to be overridden by subclasses or called by anything outside of this class
-    
-    @classproperty
-    def name(cls) -> str:
-        return cls.__name__  # type: ignore
-    
-    def __str__(self) -> str:
-        return repr(self)
-    
-    def __repr__(self) -> str:
-        """-> FaviconActor[pid=1234]"""
-        label = 'pid' if self.mode == 'process' else 'tid'
-        return f'[underline]{self.name}[/underline]\\[{label}={self.pid}]'
-    
-    @staticmethod
-    def _state_to_str(state: ObjectState) -> str:
-        """Convert a statemachine.State, models.TextChoices.choices value, or Enum value to a str"""
-        return str(state.value) if isinstance(state, State) else str(state)
-    
-    @staticmethod
-    def _sql_for_select_top_n_candidates(qs: QuerySet, claim_from_top_n: int=CLAIM_FROM_TOP_N) -> tuple[str, tuple[Any, ...]]:
-        """Get the SQL for selecting the top N candidates from the queue (to claim one from)"""
-        queryset = qs.only('id')[:claim_from_top_n]
-        select_sql, select_params = compile_sql_select(queryset)
-        return select_sql, select_params
-    
-    @staticmethod
-    def _sql_for_update_claimed_obj(qs: QuerySet, update_kwargs: dict[str, Any]) -> tuple[str, tuple[Any, ...]]:
-        """Get the SQL for updating a claimed object to mark it as ACTIVE"""
-        # qs.update(status='started', retry_at=<now + MAX_TICK_TIME>)
-        update_sql, update_params = compile_sql_update(qs, update_kwargs=update_kwargs)
-        # e.g. UPDATE core_archiveresult SET status='%s', retry_at='%s' WHERE status NOT IN ('succeeded', 'failed', 'sealed', 'started') AND retry_at <= '2024-11-04 10:14:33.240903'
-        return update_sql, update_params
-    
-    @classmethod
-    def _get_model_from_generic_typevar(cls) -> Type[ModelType]:
-        """Get the django Model from the Generic[ModelType] TypeVar arg (and check that it inherits from django.db.models.Model)"""
-        # cls.__orig_bases__ is non-standard and may be removed in the future! if this breaks,
-        # we can just require the inerited class to define the Model as a classvar manually, e.g.:
-        #     class SnapshotActor(ActorType[Snapshot]):
-        #         Model: ClassVar[Type[Snapshot]] = Snapshot
-        # https://stackoverflow.com/questions/57706180/generict-base-class-how-to-get-type-of-t-from-within-instance
-        Model = get_args(cls.__orig_bases__[0])[0]   # type: ignore
-        assert issubclass(Model, DjangoModel), f'{cls.__name__}.Model must be a valid django Model'
-        return cast(Type[ModelType], Model)
-    
-    @staticmethod
-    def _get_state_machine_cls(Model: Type[ModelType]) -> Type[StateMachine]:
-        """Get the StateMachine class for the given django Model that inherits from MachineMixin"""
-        assert issubclass(Model, MachineMixin), f'{Model.__name__} must inherit from MachineMixin and define a .state_machine_name: str'
-        model_state_machine_name = getattr(Model, 'state_machine_name', None)
-        if model_state_machine_name:
-            StateMachine = registry.get_machine_cls(model_state_machine_name)
-            assert issubclass(StateMachine, StateMachine)
-            return StateMachine
-        raise NotImplementedError(f'ActorType[{Model.__name__}] must define .state_machine_name: str that points to a valid StateMachine')
-    
-    @classmethod
-    def _get_state_machine_instance(cls, obj: ModelType) -> StateMachine:
-        """Get the StateMachine instance for the given django Model instance (and check that it is a valid instance of cls.StateMachineClass)"""
-        obj_statemachine = None
-        state_machine_attr = getattr(obj, 'state_machine_attr', 'sm')
-        try:
-            obj_statemachine = getattr(obj, state_machine_attr)
-        except Exception:
-            pass
-        
-        if not isinstance(obj_statemachine, cls.StateMachineClass):
-            raise Exception(f'{cls.__name__}: Failed to find a valid StateMachine instance at {type(obj).__name__}.{state_machine_attr}')
-            
-        return obj_statemachine
-    
-    @classmethod
-    def _populate_missing_classvars_from_model(cls, Model: Type[ModelType]):
-        """Check that the class variables are set correctly based on the ModelType"""
-        
-        # check that Model is the same as the Generic[ModelType] parameter in the class definition
-        cls.Model = getattr(cls, 'Model', None) or Model
-        if cls.Model != Model:
-            raise ValueError(f'{cls.__name__}.Model must be set to the same Model as the Generic[ModelType] parameter in the class definition')
-        
-        # check that Model has a valid StateMachine with the required event defined on it
-        cls.StateMachineClass = getattr(cls, 'StateMachineClass', None) or cls._get_state_machine_cls(cls.Model)
-        assert isinstance(cls.EVENT_NAME, str), f'{cls.__name__}.EVENT_NAME must be a str, got: {type(cls.EVENT_NAME).__name__} instead'
-        assert hasattr(cls.StateMachineClass, cls.EVENT_NAME), f'StateMachine {cls.StateMachineClass.__name__} must define a {cls.EVENT_NAME} event ({cls.__name__}.EVENT_NAME = {cls.EVENT_NAME})'
-        
-        # check that Model uses .id as its primary key field
-        primary_key_field = cls.Model._meta.pk.name
-        if primary_key_field != 'id':
-            raise NotImplementedError(f'Actors currently only support models that use .id as their primary key field ({cls.__name__} uses {cls.__name__}.{primary_key_field} as primary key)')
-        
-        # check for STATE_FIELD_NAME classvar or set it from the model's state_field_name attr
-        if not getattr(cls, 'STATE_FIELD_NAME', None):
-            if hasattr(cls.Model, 'state_field_name'):
-                cls.STATE_FIELD_NAME = getattr(cls.Model, 'state_field_name')
-            else:
-                raise NotImplementedError(f'{cls.__name__} must define a STATE_FIELD_NAME: ClassVar[str] (e.g. "status") or have a .state_field_name attr on its Model')
-        assert isinstance(cls.STATE_FIELD_NAME, str), f'{cls.__name__}.STATE_FIELD_NAME must be a str, got: {type(cls.STATE_FIELD_NAME).__name__} instead'
-            
-        # check for FINAL_STATES classvar or set it from the model's final_states attr
-        if not getattr(cls, 'FINAL_STATES', None):
-            cls.FINAL_STATES = cls.StateMachineClass.final_states
-            if not cls.FINAL_STATES:
-                raise NotImplementedError(f'{cls.__name__} must define a non-empty FINAL_STATES: ClassVar[list[str]] (e.g. ["sealed"]) or have a {cls.Model.__name__}.state_machine_name pointing to a StateMachine that provides .final_states')
-        cls.FINAL_STATES = [cls._state_to_str(state) for state in cls.FINAL_STATES]
-        assert all(isinstance(state, str) for state in cls.FINAL_STATES), f'{cls.__name__}.FINAL_STATES must be a list[str], got: {type(cls.FINAL_STATES).__name__} instead'
-        
-        # check for ACTIVE_STATE classvar or set it from the model's active_state attr
-        if not getattr(cls, 'ACTIVE_STATE', None):
-            raise NotImplementedError(f'{cls.__name__} must define an ACTIVE_STATE: ClassVar[State] (e.g. SnapshotMachine.started) ({cls.Model.__name__}.{cls.STATE_FIELD_NAME} gets set to this value to mark objects as actively processing)')
-        assert isinstance(cls.ACTIVE_STATE, (State, str)), f'{cls.__name__}.ACTIVE_STATE must be a statemachine.State | str, got: {type(cls.ACTIVE_STATE).__name__} instead'
-        
-        # check the other ClassVar attributes for valid values
-        assert cls.CLAIM_ORDER and isinstance(cls.CLAIM_ORDER, tuple) and all(isinstance(order, str) for order in cls.CLAIM_ORDER), f'{cls.__name__}.CLAIM_ORDER must be a non-empty tuple[str, ...], got: {type(cls.CLAIM_ORDER).__name__} instead'
-        assert cls.CLAIM_FROM_TOP_N > 0, f'{cls.__name__}.CLAIM_FROM_TOP_N must be a positive int, got: {cls.CLAIM_FROM_TOP_N} instead'
-        assert cls.MAX_TICK_TIME >= 1, f'{cls.__name__}.MAX_TICK_TIME must be a positive int > 1, got: {cls.MAX_TICK_TIME} instead'
-        assert cls.MAX_CONCURRENT_ACTORS >= 1, f'{cls.__name__}.MAX_CONCURRENT_ACTORS must be a positive int >=1, got: {cls.MAX_CONCURRENT_ACTORS} instead'
-        assert isinstance(cls.CLAIM_ATOMIC, bool), f'{cls.__name__}.CLAIM_ATOMIC must be a bool, got: {cls.CLAIM_ATOMIC} instead'
-
-    @classmethod
-    def _fork_actor_as_thread(cls, **launch_kwargs: LaunchKwargs) -> int:
-        """Spawn a new background thread running the actor's runloop"""
-        actor = cls(mode='thread', **launch_kwargs)
-        bg_actor_thread = Thread(target=actor.runloop)
-        bg_actor_thread.start()
-        assert bg_actor_thread.native_id is not None
-        return bg_actor_thread.native_id
-    
-    @classmethod
-    def _fork_actor_as_process(cls, **launch_kwargs: LaunchKwargs) -> int:
-        """Spawn a new background process running the actor's runloop"""
-        actor = cls(mode='process', **launch_kwargs)
-        bg_actor_process = Process(target=actor.runloop)
-        bg_actor_process.start()
-        assert bg_actor_process.pid is not None
-        cls._SPAWNED_ACTOR_PIDS.append(psutil.Process(pid=bg_actor_process.pid))
-        return bg_actor_process.pid
-    
-    
-    ### Class Methods: Called by Orchestrator on ActorType class before it has been spawned
-    
-    @classmethod
-    def get_running_actors(cls) -> list[int]:
-        """returns a list of pids of all running actors of this type"""
-        # WARNING: only works for process actors, not thread actors
-        if cls.mode == 'thread':
-            raise NotImplementedError('get_running_actors() is not implemented for thread actors')
-        return [
-            proc.pid for proc in cls._SPAWNED_ACTOR_PIDS
-            if proc.is_running() and proc.status() != 'zombie'
-        ]
-        
-    @classmethod
-    def get_actors_to_spawn(cls, queue: QuerySet, running_actors: list[int]) -> list[LaunchKwargs]:
-        """Get a list of launch kwargs for the number of actors to spawn based on the queue and currently running actors"""
-        queue_length = queue.count()
-        if not queue_length:                                      # queue is empty, spawn 0 actors
-            return []
-        
-        # WARNING:
-        # spawning new actors processes is slow/expensive, avoid spawning many actors at once in a single orchestrator tick.
-        # limit to spawning 1 or 2 at a time per orchestrator tick, and let the next tick handle starting another couple.
-        # DONT DO THIS:
-        # if queue_length > 20:                      # queue is extremely long, spawn maximum actors at once!
-        #   num_to_spawn_this_tick = cls.MAX_CONCURRENT_ACTORS
-        
-        if queue_length > 10:    
-            num_to_spawn_this_tick = 2  # spawn more actors per tick if queue is long
-        else:
-            num_to_spawn_this_tick = 1  # spawn fewer actors per tick if queue is short
-        
-        num_remaining = cls.MAX_CONCURRENT_ACTORS - len(running_actors)
-        num_to_spawn_now: int = limit(num_to_spawn_this_tick, num_remaining)
-        
-        actors_launch_kwargs: list[LaunchKwargs] = num_to_spawn_now * [{**cls.launch_kwargs}]
-        return actors_launch_kwargs
-        
-    @classmethod
-    def start(cls, mode: Literal['thread', 'process']='process', **launch_kwargs: LaunchKwargs) -> int:
-        if mode == 'thread':
-            return cls._fork_actor_as_thread(**launch_kwargs)
-        elif mode == 'process':
-            return cls._fork_actor_as_process(**launch_kwargs)
-        raise ValueError(f'Invalid actor mode: {mode} must be "thread" or "process"')
-    
-    @classproperty
-    def qs(cls) -> QuerySet[ModelType]:
-        """
-        Get the unfiltered and unsorted QuerySet of all objects that this Actor might care about.
-        Override this in the subclass to define the QuerySet of objects that the Actor is going to poll for new work.
-        (don't limit, order, or filter this by retry_at or status yet, Actor.get_queue() handles that part)
-        """
-        return cls.Model.objects.all()
-    
-    @classproperty
-    def final_q(cls) -> Q:
-        """Get the filter for objects that are in a final state"""
-        return Q(**{f'{cls.STATE_FIELD_NAME}__in': [cls._state_to_str(s) for s in cls.FINAL_STATES]})
-    
-    @classproperty
-    def active_q(cls) -> Q:
-        """Get the filter for objects that are actively processing right now"""
-        return Q(**{cls.STATE_FIELD_NAME: cls._state_to_str(cls.ACTIVE_STATE)})   # e.g. Q(status='started')
-    
-    @classproperty
-    def stalled_q(cls) -> Q:
-        """Get the filter for objects that are marked active but have timed out"""
-        return cls.active_q & Q(retry_at__lte=timezone.now())                     # e.g. Q(status='started') AND Q(<retry_at is in the past>)
-    
-    @classproperty
-    def future_q(cls) -> Q:
-        """Get the filter for objects that have a retry_at in the future"""
-        return Q(retry_at__gt=timezone.now())
-    
-    @classproperty
-    def pending_q(cls) -> Q:
-        """Get the filter for objects that are ready for processing."""
-        return ~(cls.active_q) & ~(cls.final_q) & ~(cls.future_q)
-    
-    @classmethod
-    def get_queue(cls, sort: bool=True) -> QuerySet[ModelType]:
-        """
-        Get the sorted and filtered QuerySet of objects that are ready for processing.
-        e.g. qs.exclude(status__in=('sealed', 'started'), retry_at__gt=timezone.now()).order_by('retry_at')
-        """
-        unsorted_qs = cls.qs.filter(cls.pending_q)
-        return unsorted_qs.order_by(*cls.CLAIM_ORDER) if sort else unsorted_qs
-
-    ### Instance Methods: Only called from within Actor instance after it has been spawned (i.e. forked as a thread or process)
-    
-    def runloop(self):
-        """The main runloop that starts running when the actor is spawned (as subprocess or thread) and exits when the queue is empty"""
-        self.on_startup()
-        obj_to_process: ModelType | None = None
-        last_error: BaseException | None = None
-        try:
-            while True:
-                # Get the next object to process from the queue
-                try:
-                    obj_to_process = cast(ModelType, self.get_next(atomic=self.atomic))
-                except (ActorQueueIsEmpty, ActorObjectAlreadyClaimed) as err:
-                    last_error = err
-                    obj_to_process = None
-                
-                # Handle the case where there is no next object to process
-                if obj_to_process:
-                    self.idle_count = 0   # reset idle count if we got an object
-                else:
-                    if self.idle_count >= 30:
-                        break             # stop looping and exit if queue is empty and we have idled for 30sec
-                    else:
-                        # print('Actor runloop()', f'pid={self.pid}', 'queue empty, rechecking...')
-                        self.idle_count += 1
-                        time.sleep(1)
-                        continue
-                
-                # Process the object by triggering its StateMachine.tick() method
-                self.on_tick_start(obj_to_process)
-                try:
-                    self.tick(obj_to_process)
-                except Exception as err:
-                    last_error = err
-                    # print(f'[red]🏃‍♂️ {self}.tick()[/red] {obj_to_process} ERROR: [red]{type(err).__name__}: {err}[/red]')
-                    db.connections.close_all()                         # always reset the db connection after an exception to clear any pending transactions
-                    self.on_tick_exception(obj_to_process, err)
-                finally:
-                    self.on_tick_end(obj_to_process)
-
-        except BaseException as err:
-            last_error = err
-            if isinstance(err, KeyboardInterrupt):
-                print()
-            else:
-                print(f'\n[red]🏃‍♂️ {self}.runloop() FATAL:[/red] {type(err).__name__}: {err}')
-                print(f'    Last processed object: {obj_to_process}')
-                raise
-        finally:
-            self.on_shutdown(last_obj=obj_to_process, last_error=last_error)
-    
-    def get_update_kwargs_to_claim_obj(self) -> dict[str, Any]:
-        """
-        Get the field values needed to mark an pending obj_to_process as being actively processing (aka claimed)
-        by the current Actor. returned kwargs will be applied using: qs.filter(id=obj_to_process.id).update(**kwargs).
-        F() expressions are allowed in field values if you need to update a field based on its current value.
-        Can be a defined as a normal method (instead of classmethod) on subclasses if it needs to access instance vars.
-        """
-        return {
-            self.STATE_FIELD_NAME: self.ACTIVE_STATE,
-            'retry_at': timezone.now() + timedelta(seconds=self.MAX_TICK_TIME),
-        }
-    
-    def get_next(self, atomic: bool | None=None) -> ModelType | None:
-        """get the next object from the queue, atomically locking it if self.atomic=True"""
-        atomic = self.CLAIM_ATOMIC if atomic is None else atomic
-        if atomic:
-            # fetch and claim the next object from in the queue in one go atomically
-            obj = self.get_next_atomic()
-        else:
-            # two-step claim: fetch the next object and lock it in a separate query
-            obj = self.get_next_non_atomic()
-        return obj
-    
-    def get_next_non_atomic(self) -> ModelType:
-        """
-        Naiively selects the top/first object from self.get_queue().order_by(*self.CLAIM_ORDER),
-        then claims it by running .update(status='started', retry_at=<now + MAX_TICK_TIME>).
-        
-        Do not use this method if there is more than one Actor racing to get objects from the same queue,
-        it will be slow/buggy as they'll compete to lock the same object at the same time (TOCTTOU race).
-        """
-        obj = self.get_queue().first()
-        if obj is None:
-            raise ActorQueueIsEmpty(f'No next object available in {self}.get_queue()')
-        
-        locked = self.get_queue().filter(id=obj.id).update(**self.get_update_kwargs_to_claim_obj())
-        if not locked:
-            raise ActorObjectAlreadyClaimed(f'Unable to lock the next {self.Model.__name__} object from {self}.get_queue().first()')
-        return obj
-        
-    def get_next_atomic(self) -> ModelType | None:
-        """
-        Selects the top n=50 objects from the queue and atomically claims a random one from that set.
-        This approach safely minimizes contention with other Actors trying to select from the same Queue.
-
-        The atomic query is roughly equivalent to the following:  (all done in one SQL query to avoid a TOCTTOU race)
-            top_candidates are selected from:   qs.order_by(*CLAIM_ORDER).only('id')[:CLAIM_FROM_TOP_N]
-            a single candidate is chosen using: qs.filter(id__in=top_n_candidates).order_by('?').first()
-            the chosen obj is claimed using:    qs.filter(id=chosen_obj).update(status=ACTIVE_STATE, retry_at=<now + MAX_TICK_TIME>)
-        """
-        # TODO: if we switch from SQLite to PostgreSQL in the future, we should change this
-        # to use SELECT FOR UPDATE instead of a subquery + ORDER BY RANDOM() LIMIT 1
-        
-        # e.g. SELECT id FROM core_archiveresult WHERE status NOT IN (...) AND retry_at <= '...' ORDER BY retry_at ASC LIMIT 50
-        qs = self.get_queue()
-        select_top_canidates_sql, select_params = self._sql_for_select_top_n_candidates(qs=qs)
-        assert select_top_canidates_sql.startswith('SELECT ')
-        
-        # e.g. UPDATE core_archiveresult SET status='%s', retry_at='%s' WHERE status NOT IN (...) AND retry_at <= '...'
-        update_claimed_obj_sql, update_params = self._sql_for_update_claimed_obj(qs=qs, update_kwargs=self.get_update_kwargs_to_claim_obj())
-        assert update_claimed_obj_sql.startswith('UPDATE ')
-        db_table = self.Model._meta.db_table  # e.g. core_archiveresult
-        
-        # subquery gets the pool of the top candidates e.g. self.get_queue().only('id')[:CLAIM_FROM_TOP_N]
-        # main query selects a random one from that pool, and claims it using .update(status=ACTIVE_STATE, retry_at=<now + MAX_TICK_TIME>)
-        # this is all done in one atomic SQL query to avoid TOCTTOU race conditions (as much as possible)
-        atomic_select_and_update_sql = f"""
-            {update_claimed_obj_sql} AND "{db_table}"."id" = (
-                SELECT "{db_table}"."id" FROM (
-                    {select_top_canidates_sql}
-                ) candidates
-                ORDER BY RANDOM()
-                LIMIT 1
-            )
-            RETURNING *;
-        """
-        try:
-            return self.Model.objects.raw(atomic_select_and_update_sql, (*update_params, *select_params))[0]
-        except KeyError:
-            if self.get_queue().exists():
-                raise ActorObjectAlreadyClaimed(f'Unable to lock the next {self.Model.__name__} object from {self}.get_queue().first()')
-            else:
-                raise ActorQueueIsEmpty(f'No next object available in {self}.get_queue()')
-
-    def tick(self, obj_to_process: ModelType) -> None:
-        """Call the object.sm.tick() method to process the object"""
-        print(f'[blue]🏃‍♂️ {self}.tick()[/blue] {obj_to_process}')
-        
-        # get the StateMachine instance from the object
-        obj_statemachine = self._get_state_machine_instance(obj_to_process)
-        
-        # trigger the event on the StateMachine instance
-        obj_tick_method = getattr(obj_statemachine, self.EVENT_NAME)  # e.g. obj_statemachine.tick()
-        obj_tick_method()
-        
-        # save the object to persist any state changes
-        obj_to_process.save()
-        
-    def on_startup(self) -> None:
-        if self.mode == 'thread':
-            self.pid = get_native_id()  # thread id
-            print(f'[green]🏃‍♂️ {self}.on_startup() STARTUP (THREAD)[/green]')
-        else:
-            self.pid = os.getpid()      # process id
-            print(f'[green]🏃‍♂️ {self}.on_startup() STARTUP (PROCESS)[/green]')
-        # abx.pm.hook.on_actor_startup(actor=self)
-        
-    def on_shutdown(self, last_obj: ModelType | None=None, last_error: BaseException | None=None) -> None:
-        if isinstance(last_error, KeyboardInterrupt) or last_error is None:
-            last_error_str = '[green](CTRL-C)[/green]'
-        elif isinstance(last_error, ActorQueueIsEmpty):
-            last_error_str = '[green](queue empty)[/green]'
-        elif isinstance(last_error, ActorObjectAlreadyClaimed):
-            last_error_str = '[green](queue race)[/green]'
-        else:
-            last_error_str = f'[red]{type(last_error).__name__}: {last_error}[/red]'
-
-        print(f'[grey53]🏃‍♂️ {self}.on_shutdown() SHUTTING DOWN[/grey53] {last_error_str}')
-        # abx.pm.hook.on_actor_shutdown(actor=self, last_obj=last_obj, last_error=last_error)
-        
-    def on_tick_start(self, obj_to_process: ModelType) -> None:
-        print(f'🏃‍♂️ {self}.on_tick_start() {obj_to_process}')
-        # abx.pm.hook.on_actor_tick_start(actor=self, obj_to_process=obj)
-        # self.timer = TimedProgress(self.MAX_TICK_TIME, prefix='      ')
-    
-    def on_tick_end(self, obj_to_process: ModelType) -> None:
-        print(f'🏃‍♂️ {self}.on_tick_end() {obj_to_process}')
-        # abx.pm.hook.on_actor_tick_end(actor=self, obj_to_process=obj_to_process)
-        # self.timer.end()
-    
-    def on_tick_exception(self, obj_to_process: ModelType, error: Exception) -> None:
-        print(f'[red]🏃‍♂️ {self}.on_tick_exception()[/red] {obj_to_process}: [red]{type(error).__name__}: {error}[/red]')
-        # abx.pm.hook.on_actor_tick_exception(actor=self, obj_to_process=obj_to_process, error=error)
-
-
-
-def compile_sql_select(queryset: QuerySet, filter_kwargs: dict[str, Any] | None=None, order_args: tuple[str, ...]=(), limit: int | None=None) -> tuple[str, tuple[Any, ...]]:
-    """
-    Compute the SELECT query SQL for a queryset.filter(**filter_kwargs).order_by(*order_args)[:limit] call
-    Returns a tuple of (sql, params) where sql is a template string containing %s (unquoted) placeholders for the params
-    
-    WARNING:
-    final_sql = sql % params  DOES NOT WORK to assemble the final SQL string because the %s placeholders are not quoted/escaped
-    they should always passed separately to the DB driver so it can do its own quoting/escaping to avoid SQL injection and syntax errors
-    """
-    assert isinstance(queryset, QuerySet), f'compile_sql_select(...) first argument must be a QuerySet, got: {type(queryset).__name__} instead'
-    assert filter_kwargs is None or isinstance(filter_kwargs, dict), f'compile_sql_select(...) filter_kwargs argument must be a dict[str, Any], got: {type(filter_kwargs).__name__} instead'
-    assert isinstance(order_args, tuple) and all(isinstance(arg, str) for arg in order_args), f'compile_sql_select(...) order_args argument must be a tuple[str, ...] got: {type(order_args).__name__} instead'
-    assert limit is None or isinstance(limit, int), f'compile_sql_select(...) limit argument must be an int, got: {type(limit).__name__} instead'
-    
-    queryset = queryset._chain()                      # type: ignore   # copy queryset to avoid modifying the original
-    if filter_kwargs:
-        queryset = queryset.filter(**filter_kwargs)
-    if order_args:
-        queryset = queryset.order_by(*order_args)
-    if limit is not None:
-        queryset = queryset[:limit]
-    query = queryset.query
-    
-    # e.g. SELECT id FROM core_archiveresult WHERE status NOT IN (%s, %s, %s) AND retry_at <= %s ORDER BY retry_at ASC LIMIT 50
-    select_sql, select_params = query.get_compiler(queryset.db).as_sql()
-    return select_sql, select_params
-
-
-def compile_sql_update(queryset: QuerySet, update_kwargs: dict[str, Any], filter_kwargs: dict[str, Any] | None=None) -> tuple[str, tuple[Any, ...]]:
-    """
-    Compute the UPDATE query SQL for a queryset.filter(**filter_kwargs).update(**update_kwargs) call
-    Returns a tuple of (sql, params) where sql is a template string containing %s (unquoted) placeholders for the params
-    
-    Based on the django.db.models.QuerySet.update() source code, but modified to return the SQL instead of executing the update
-    https://github.com/django/django/blob/611bf6c2e2a1b4ab93273980c45150c099ab146d/django/db/models/query.py#L1217
-    
-    WARNING:
-    final_sql = sql % params  DOES NOT WORK to assemble the final SQL string because the %s placeholders are not quoted/escaped
-    they should always passed separately to the DB driver so it can do its own quoting/escaping to avoid SQL injection and syntax errors
-    """
-    assert isinstance(queryset, QuerySet), f'compile_sql_update(...) first argument must be a QuerySet, got: {type(queryset).__name__} instead'
-    assert isinstance(update_kwargs, dict), f'compile_sql_update(...) update_kwargs argument must be a dict[str, Any], got: {type(update_kwargs).__name__} instead'
-    assert filter_kwargs is None or isinstance(filter_kwargs, dict), f'compile_sql_update(...) filter_kwargs argument must be a dict[str, Any], got: {type(filter_kwargs).__name__} instead'
-    
-    queryset = queryset._chain()                      # type: ignore   # copy queryset to avoid modifying the original
-    if filter_kwargs:
-        queryset = queryset.filter(**filter_kwargs)
-    queryset.query.clear_ordering(force=True)                          # clear any ORDER BY clauses
-    queryset.query.clear_limits()                                      # clear any LIMIT clauses aka slices[:n]
-    queryset._for_write = True                        # type: ignore
-    query = queryset.query.chain(sql.UpdateQuery)     # type: ignore
-    query.add_update_values(update_kwargs)            # type: ignore
-    query.annotations = {}                                             # clear any annotations
-    
-    # e.g. UPDATE core_archiveresult SET status='%s', retry_at='%s' WHERE status NOT IN (%s, %s, %s) AND retry_at <= %s
-    update_sql, update_params = query.get_compiler(queryset.db).as_sql()
-    return update_sql, update_params
@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.
@@ -1,300 +0,0 @@
-from typing import ClassVar, Type, Iterable
-from datetime import datetime, timedelta
-
-from statemachine.mixins import MachineMixin
-
-from django.db import models
-from django.utils import timezone
-from django.utils.functional import classproperty
-
-from statemachine import registry, StateMachine, State
-
-from django.core import checks
-
-class DefaultStatusChoices(models.TextChoices):
-    QUEUED = 'queued', 'Queued'
-    STARTED = 'started', 'Started'
-    SEALED = 'sealed', 'Sealed'
-
-
-default_status_field: models.CharField = models.CharField(choices=DefaultStatusChoices.choices, max_length=15, default=DefaultStatusChoices.QUEUED, null=False, blank=False, db_index=True)
-default_retry_at_field: models.DateTimeField = models.DateTimeField(default=timezone.now, null=False, db_index=True)
-
-ObjectState = State | str
-ObjectStateList = Iterable[ObjectState]
-
-
-class BaseModelWithStateMachine(models.Model, MachineMixin):
-    id: models.UUIDField
-    
-    StatusChoices: ClassVar[Type[models.TextChoices]]
-    
-    # status: models.CharField
-    # retry_at: models.DateTimeField
-    
-    state_machine_name: ClassVar[str]
-    state_field_name: ClassVar[str]
-    state_machine_attr: ClassVar[str] = 'sm'
-    bind_events_as_methods: ClassVar[bool] = True
-    
-    active_state: ClassVar[ObjectState]
-    retry_at_field_name: ClassVar[str]
-    
-    class Meta:
-        abstract = True
-        
-    @classmethod
-    def check(cls, sender=None, **kwargs):
-        errors = super().check(**kwargs)
-        
-        found_id_field = False
-        found_status_field = False
-        found_retry_at_field = False
-        
-        for field in cls._meta.get_fields():
-            if getattr(field, '_is_state_field', False):
-                if cls.state_field_name == field.name:
-                    found_status_field = True
-                    if getattr(field, 'choices', None) != cls.StatusChoices.choices:
-                        errors.append(checks.Error(
-                            f'{cls.__name__}.{field.name} must have choices set to {cls.__name__}.StatusChoices.choices',
-                            hint=f'{cls.__name__}.{field.name}.choices = {getattr(field, "choices", None)!r}',
-                            obj=cls,
-                            id='actors.E011',
-                        ))
-            if getattr(field, '_is_retry_at_field', False):
-                if cls.retry_at_field_name == field.name:
-                    found_retry_at_field = True
-            if field.name == 'id' and getattr(field, 'primary_key', False):
-                found_id_field = True
-                    
-        if not found_status_field:
-            errors.append(checks.Error(
-                f'{cls.__name__}.state_field_name must be defined and point to a StatusField()',
-                hint=f'{cls.__name__}.state_field_name = {cls.state_field_name!r} but {cls.__name__}.{cls.state_field_name!r} was not found or does not refer to StatusField',
-                obj=cls,
-                id='actors.E012',
-            ))
-        if not found_retry_at_field:
-            errors.append(checks.Error(
-                f'{cls.__name__}.retry_at_field_name must be defined and point to a RetryAtField()',
-                hint=f'{cls.__name__}.retry_at_field_name = {cls.retry_at_field_name!r} but {cls.__name__}.{cls.retry_at_field_name!r} was not found or does not refer to RetryAtField',
-                obj=cls,
-                id='actors.E013',
-            ))
-            
-        if not found_id_field:
-            errors.append(checks.Error(
-                f'{cls.__name__} must have an id field that is a primary key',
-                hint=f'{cls.__name__}.id = {cls.id!r}',
-                obj=cls,
-                id='actors.E014',
-            ))
-            
-        if not isinstance(cls.state_machine_name, str):
-            errors.append(checks.Error(
-                f'{cls.__name__}.state_machine_name must be a dotted-import path to a StateMachine class',
-                hint=f'{cls.__name__}.state_machine_name = {cls.state_machine_name!r}',
-                obj=cls,
-                id='actors.E015',
-            ))
-        
-        try:
-            cls.StateMachineClass
-        except Exception as err:
-            errors.append(checks.Error(
-                f'{cls.__name__}.state_machine_name must point to a valid StateMachine class, but got {type(err).__name__} {err} when trying to access {cls.__name__}.StateMachineClass',
-                hint=f'{cls.__name__}.state_machine_name = {cls.state_machine_name!r}',
-                obj=cls,
-                id='actors.E016',
-            ))
-        
-        if cls.INITIAL_STATE not in cls.StatusChoices.values:
-            errors.append(checks.Error(
-                f'{cls.__name__}.StateMachineClass.initial_state must be present within {cls.__name__}.StatusChoices',
-                hint=f'{cls.__name__}.StateMachineClass.initial_state = {cls.StateMachineClass.initial_state!r}',
-                obj=cls,
-                id='actors.E017',
-            ))
-            
-        if cls.ACTIVE_STATE not in cls.StatusChoices.values:
-            errors.append(checks.Error(
-                f'{cls.__name__}.active_state must be set to a valid State present within {cls.__name__}.StatusChoices',
-                hint=f'{cls.__name__}.active_state = {cls.active_state!r}',
-                obj=cls,
-                id='actors.E018',
-            ))
-            
-        
-        for state in cls.FINAL_STATES:
-            if state not in cls.StatusChoices.values:
-                errors.append(checks.Error(
-                    f'{cls.__name__}.StateMachineClass.final_states must all be present within {cls.__name__}.StatusChoices',
-                    hint=f'{cls.__name__}.StateMachineClass.final_states = {cls.StateMachineClass.final_states!r}',
-                    obj=cls,
-                    id='actors.E019',
-                ))
-                break
-        return errors
-    
-    @staticmethod
-    def _state_to_str(state: ObjectState) -> str:
-        """Convert a statemachine.State, models.TextChoices.choices value, or Enum value to a str"""
-        return str(state.value) if isinstance(state, State) else str(state)
-    
-    
-    @property
-    def RETRY_AT(self) -> datetime:
-        return getattr(self, self.retry_at_field_name)
-    
-    @RETRY_AT.setter
-    def RETRY_AT(self, value: datetime):
-        setattr(self, self.retry_at_field_name, value)
-        
-    @property
-    def STATE(self) -> str:
-        return getattr(self, self.state_field_name)
-    
-    @STATE.setter
-    def STATE(self, value: str):
-        setattr(self, self.state_field_name, value)
-        
-    def bump_retry_at(self, seconds: int = 10):
-        self.RETRY_AT = timezone.now() + timedelta(seconds=seconds)
-        
-    @classproperty
-    def ACTIVE_STATE(cls) -> str:
-        return cls._state_to_str(cls.StateMachineClass.active_state)
-        
-    @classproperty
-    def INITIAL_STATE(cls) -> str:
-        return cls._state_to_str(cls.StateMachineClass.initial_state)
-    
-    @classproperty
-    def FINAL_STATES(cls) -> list[str]:
-        return [cls._state_to_str(state) for state in cls.StateMachineClass.final_states]
-    
-    @classproperty
-    def FINAL_OR_ACTIVE_STATES(cls) -> list[str]:
-        return [*cls.FINAL_STATES, cls.ACTIVE_STATE]
-        
-    @classmethod
-    def extend_choices(cls, base_choices: Type[models.TextChoices]):
-        """
-        Decorator to extend the base choices with extra choices, e.g.:
-        
-        class MyModel(ModelWithStateMachine):
-        
-            @ModelWithStateMachine.extend_choices(ModelWithStateMachine.StatusChoices)
-            class StatusChoices(models.TextChoices):
-                SUCCEEDED = 'succeeded'
-                FAILED = 'failed'
-                SKIPPED = 'skipped'
-        """
-        assert issubclass(base_choices, models.TextChoices), f'@extend_choices(base_choices) must be a TextChoices class, not {base_choices.__name__}'
-        def wrapper(extra_choices: Type[models.TextChoices]) -> Type[models.TextChoices]:
-            joined = {}
-            for item in base_choices.choices:
-                joined[item[0]] = item[1]
-            for item in extra_choices.choices:
-                joined[item[0]] = item[1]
-            return models.TextChoices('StatusChoices', joined)
-        return wrapper
-        
-    @classmethod
-    def StatusField(cls, **kwargs) -> models.CharField:
-        """
-        Used on subclasses to extend/modify the status field with updated kwargs. e.g.:
-        
-        class MyModel(ModelWithStateMachine):
-            class StatusChoices(ModelWithStateMachine.StatusChoices):
-                QUEUED = 'queued', 'Queued'
-                STARTED = 'started', 'Started'
-                SEALED = 'sealed', 'Sealed'
-                BACKOFF = 'backoff', 'Backoff'
-                FAILED = 'failed', 'Failed'
-                SKIPPED = 'skipped', 'Skipped'
-        
-            status = ModelWithStateMachine.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED)
-        """
-        default_kwargs = default_status_field.deconstruct()[3]
-        updated_kwargs = {**default_kwargs, **kwargs}
-        field = models.CharField(**updated_kwargs)
-        field._is_state_field = True                    # type: ignore
-        return field
-
-    @classmethod
-    def RetryAtField(cls, **kwargs) -> models.DateTimeField:
-        """
-        Used on subclasses to extend/modify the retry_at field with updated kwargs. e.g.:
-        
-        class MyModel(ModelWithStateMachine):
-            retry_at = ModelWithStateMachine.RetryAtField(editable=False)
-        """
-        default_kwargs = default_retry_at_field.deconstruct()[3]
-        updated_kwargs = {**default_kwargs, **kwargs}
-        field = models.DateTimeField(**updated_kwargs)
-        field._is_retry_at_field = True                 # type: ignore
-        return field
-    
-    @classproperty
-    def StateMachineClass(cls) -> Type[StateMachine]:
-        """Get the StateMachine class for the given django Model that inherits from MachineMixin"""
-
-        model_state_machine_name = getattr(cls, 'state_machine_name', None)
-        if model_state_machine_name:
-            StateMachineCls = registry.get_machine_cls(model_state_machine_name)
-            assert issubclass(StateMachineCls, StateMachine)
-            return StateMachineCls
-        raise NotImplementedError(f'ActorType[{cls.__name__}] must define .state_machine_name: str that points to a valid StateMachine')
-    
-    # @classproperty
-    # def final_q(cls) -> Q:
-    #     """Get the filter for objects that are in a final state"""
-    #     return Q(**{f'{cls.state_field_name}__in': cls.final_states})
-    
-    # @classproperty
-    # def active_q(cls) -> Q:
-    #     """Get the filter for objects that are actively processing right now"""
-    #     return Q(**{cls.state_field_name: cls._state_to_str(cls.active_state)})   # e.g. Q(status='started')
-    
-    # @classproperty
-    # def stalled_q(cls) -> Q:
-    #     """Get the filter for objects that are marked active but have timed out"""
-    #     return cls.active_q & Q(retry_at__lte=timezone.now())                     # e.g. Q(status='started') AND Q(<retry_at is in the past>)
-    
-    # @classproperty
-    # def future_q(cls) -> Q:
-    #     """Get the filter for objects that have a retry_at in the future"""
-    #     return Q(retry_at__gt=timezone.now())
-    
-    # @classproperty
-    # def pending_q(cls) -> Q:
-    #     """Get the filter for objects that are ready for processing."""
-    #     return ~(cls.active_q) & ~(cls.final_q) & ~(cls.future_q)
-    
-    # @classmethod
-    # def get_queue(cls) -> QuerySet:
-    #     """
-    #     Get the sorted and filtered QuerySet of objects that are ready for processing.
-    #     e.g. qs.exclude(status__in=('sealed', 'started'), retry_at__gt=timezone.now()).order_by('retry_at')
-    #     """
-    #     return cls.objects.filter(cls.pending_q)
-
-
-class ModelWithStateMachine(BaseModelWithStateMachine):
-    StatusChoices: ClassVar[Type[DefaultStatusChoices]] = DefaultStatusChoices
-    
-    status: models.CharField = BaseModelWithStateMachine.StatusField()
-    retry_at: models.DateTimeField = BaseModelWithStateMachine.RetryAtField()
-    
-    state_machine_name: ClassVar[str]      # e.g. 'core.statemachines.ArchiveResultMachine'
-    state_field_name: ClassVar[str]        = 'status'
-    state_machine_attr: ClassVar[str]      = 'sm'
-    bind_events_as_methods: ClassVar[bool] = True
-    
-    active_state: ClassVar[str]            = StatusChoices.STARTED
-    retry_at_field_name: ClassVar[str]     = 'retry_at'
-    
-    class Meta:
-        abstract = True
@@ -1,244 +0,0 @@
-__package__ = 'archivebox.actors'
-
-import os
-import time
-import itertools
-from typing import Dict, Type, Literal, ClassVar
-from django.utils.functional import classproperty
-
-from multiprocessing import Process
-from threading import Thread, get_native_id
-
-
-from rich import print
-
-from django.db.models import QuerySet
-
-from django.apps import apps
-from .actor import ActorType
-
-class Orchestrator:
-    pid: int
-    idle_count: int = 0
-    actor_types: Dict[str, Type[ActorType]]
-    mode: Literal['thread', 'process'] = 'process'
-
-    def __init__(self, actor_types: Dict[str, Type[ActorType]] | None = None, mode: Literal['thread', 'process'] | None=None):
-        self.actor_types = actor_types or self.actor_types or self.autodiscover_actor_types()
-        self.mode = mode or self.mode
-
-    def __repr__(self) -> str:
-        label = 'tid' if self.mode == 'thread' else 'pid'
-        return f'[underline]{self.name}[/underline]\\[{label}={self.pid}]'
-    
-    def __str__(self) -> str:
-        return self.__repr__()
-    
-    @classproperty
-    def name(cls) -> str:
-        return cls.__name__   # type: ignore
-    
-    def fork_as_thread(self):
-        self.thread = Thread(target=self.runloop)
-        self.thread.start()
-        assert self.thread.native_id is not None
-        return self.thread.native_id
-    
-    def fork_as_process(self):
-        self.process = Process(target=self.runloop)
-        self.process.start()
-        assert self.process.pid is not None
-        return self.process.pid
-
-    def start(self) -> int:
-        if self.mode == 'thread':
-            return self.fork_as_thread()
-        elif self.mode == 'process':
-            return self.fork_as_process()
-        raise ValueError(f'Invalid orchestrator mode: {self.mode}')
-    
-    @classmethod
-    def autodiscover_actor_types(cls) -> Dict[str, Type[ActorType]]:
-        # returns a Dict of all discovered {actor_type_id: ActorType} across the codebase
-        # override this method in a subclass to customize the actor types that are used
-        # return {'Snapshot': SnapshotActorType, 'ArchiveResult_chrome': ChromeActorType, ...}
-        return {
-            # look through all models and find all classes that inherit from ActorType
-            # actor_type.__name__: actor_type
-            # for actor_type in abx.pm.hook.get_all_ACTORS_TYPES().values()
-        }
-    
-    @classmethod
-    def get_orphaned_objects(cls, all_queues) -> list:
-        # returns a list of objects that are in the queues of all actor types but not in the queues of any other actor types
-        all_queued_ids = itertools.chain(*[queue.values('id', flat=True) for queue in all_queues.values()])
-        orphaned_objects = []
-        for model in apps.get_models():
-            if hasattr(model, 'retry_at'):
-                orphaned_objects.extend(model.objects.filter(retry_at__lt=timezone.now()).exclude(id__in=all_queued_ids))
-        return orphaned_objects
-    
-    def on_startup(self):
-        if self.mode == 'thread':
-            self.pid = get_native_id()
-            print(f'[green]👨‍✈️ {self}.on_startup() STARTUP (THREAD)[/green]')
-        elif self.mode == 'process':
-            self.pid = os.getpid()
-            print(f'[green]👨‍✈️ {self}.on_startup() STARTUP (PROCESS)[/green]')
-        # abx.pm.hook.on_orchestrator_startup(self)
-    
-    def on_shutdown(self, err: BaseException | None = None):
-        print(f'[grey53]👨‍✈️ {self}.on_shutdown() SHUTTING DOWN[/grey53]', err or '[green](gracefully)[/green]')
-        # abx.pm.hook.on_orchestrator_shutdown(self)
-        
-    def on_tick_started(self, all_queues):
-        # total_pending = sum(queue.count() for queue in all_queues.values())
-        # print(f'👨‍✈️ {self}.on_tick_started()', f'total_pending={total_pending}')
-        # abx.pm.hook.on_orchestrator_tick_started(self, actor_types, all_queues)
-        pass
-    
-    def on_tick_finished(self, all_queues, all_existing_actors, all_spawned_actors):
-        if all_spawned_actors:
-            total_queue_length = sum(queue.count() for queue in all_queues.values())
-            print(f'[grey53]👨‍✈️ {self}.on_tick_finished() queue={total_queue_length} existing_actors={len(all_existing_actors)} spawned_actors={len(all_spawned_actors)}[/grey53]')
-        # abx.pm.hook.on_orchestrator_tick_finished(self, actor_types, all_queues)
-
-    def on_idle(self, all_queues):
-        # print(f'👨‍✈️ {self}.on_idle()')
-        # abx.pm.hook.on_orchestrator_idle(self)
-        # check for orphaned objects left behind
-        if self.idle_count == 60:
-            orphaned_objects = self.get_orphaned_objects(all_queues)
-            if orphaned_objects:
-                print('[red]👨‍✈️ WARNING: some objects may not be processed, no actor has claimed them after 60s:[/red]', orphaned_objects)
-
-    def runloop(self):
-        self.on_startup()
-        try:
-            while True:
-                all_queues = {
-                    actor_type: actor_type.get_queue()
-                    for actor_type in self.actor_types.values()
-                }
-                if not all_queues:
-                    raise Exception('Failed to find any actor_types to process')
-
-                self.on_tick_started(all_queues)
-
-                all_existing_actors = []
-                all_spawned_actors = []
-
-                for actor_type, queue in all_queues.items():
-                    try:
-                        existing_actors = actor_type.get_running_actors()
-                        all_existing_actors.extend(existing_actors)
-                        actors_to_spawn = actor_type.get_actors_to_spawn(queue, existing_actors)
-                        for launch_kwargs in actors_to_spawn:
-                            new_actor_pid = actor_type.start(mode='process', **launch_kwargs)
-                            all_spawned_actors.append(new_actor_pid)
-                    except Exception as err:
-                        print(f'🏃‍♂️ ERROR: {self} Failed to get {actor_type} queue & running actors', err)
-                    except BaseException:
-                        raise
-
-                if not any(queue.exists() for queue in all_queues.values()):
-                    self.on_idle(all_queues)
-                    self.idle_count += 1
-                    time.sleep(1)
-                else:
-                    self.idle_count = 0
-                    
-                self.on_tick_finished(all_queues, all_existing_actors, all_spawned_actors)
-                time.sleep(1)
-
-        except BaseException as err:
-            if isinstance(err, KeyboardInterrupt):
-                print()
-            else:
-                print(f'\n[red]🏃‍♂️ {self}.runloop() FATAL:[/red]', err.__class__.__name__, err)
-            self.on_shutdown(err=err)
-
-
-
-from archivebox.config.django import setup_django
-
-setup_django()
-
-from core.models import ArchiveResult, Snapshot
-
-from django.utils import timezone
-
-from django import db
-from django.db import connection
-
-
-from crawls.actors import CrawlActor
-from .actor_snapshot import SnapshotActor
-
-from abx_plugin_singlefile.actors import SinglefileActor
-
-
-class FaviconActor(ActorType[ArchiveResult]):
-    CLAIM_ORDER: ClassVar[str] = 'created_at DESC'
-    CLAIM_WHERE: ClassVar[str] = 'status = "queued" AND extractor = "favicon"'
-    CLAIM_SET: ClassVar[str] = 'status = "started"'
-    
-    @classproperty
-    def QUERYSET(cls) -> QuerySet:
-        return ArchiveResult.objects.filter(status='failed', extractor='favicon')
-
-    def tick(self, obj: ArchiveResult):
-        print(f'[grey53]{self}.tick({obj.abid or obj.id}, status={obj.status}) remaining:[/grey53]', self.get_queue().count())
-        updated = ArchiveResult.objects.filter(id=obj.id, status='started').update(status='success') == 1
-        if not updated:
-            raise Exception(f'Failed to update {obj.abid or obj.id}, interrupted by another actor writing to the same object')
-        obj.refresh_from_db()
-        obj.save()
-
-
-class ExtractorsOrchestrator(Orchestrator):
-    actor_types = {
-        'CrawlActor': CrawlActor,
-        'SnapshotActor': SnapshotActor,
-        'FaviconActor': FaviconActor,
-        'SinglefileActor': SinglefileActor,
-    }
-
-
-if __name__ == '__main__':    
-    orchestrator = ExtractorsOrchestrator()
-    orchestrator.start()
-    
-    snap = Snapshot.objects.last()
-    assert snap is not None
-    created = 0
-    while True:
-        time.sleep(0.05)
-        # try:
-        #     ArchiveResult.objects.bulk_create([
-        #         ArchiveResult(
-        #             id=uuid.uuid4(),
-        #             snapshot=snap,
-        #             status='failed',
-        #             extractor='favicon',
-        #             cmd=['echo', '"hello"'],
-        #             cmd_version='1.0',
-        #             pwd='.',
-        #             start_ts=timezone.now(),
-        #             end_ts=timezone.now(),
-        #             created_at=timezone.now(),
-        #             modified_at=timezone.now(),
-        #             created_by_id=1,
-        #         )
-        #         for _ in range(100)
-        #     ])
-        #     created += 100
-        #     if created % 1000 == 0:
-        #         print(f'[blue]Created {created} ArchiveResults...[/blue]')
-        #         time.sleep(25)
-        # except Exception as err:
-        #     print(err)
-        #     db.connections.close_all()
-        # except BaseException as err:
-        #     print(err)
-        #     break
@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.
@@ -1,3 +0,0 @@
-from django.shortcuts import render
-
-# Create your views here.
@@ -1 +1 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"
@@ -1,31 +1,98 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"

+from django.contrib import admin
+from django.http import HttpRequest
 from signal_webhooks.admin import WebhookAdmin
 from signal_webhooks.utils import get_webhook_model

-from abid_utils.admin import ABIDModelAdmin
+from archivebox.base_models.admin import BaseModelAdmin

-from api.models import APIToken
+from archivebox.api.models import APIToken


-class APITokenAdmin(ABIDModelAdmin):
-    list_display = ('created_at', 'abid', 'created_by', 'token_redacted', 'expires')
-    sort_fields = ('abid', 'created_at', 'created_by', 'expires')
-    readonly_fields = ('created_at', 'modified_at', 'abid_info')
-    search_fields = ('id', 'abid', 'created_by__username', 'token')
-    fields = ('created_by', 'token', 'expires', *readonly_fields)
+class APITokenAdmin(BaseModelAdmin):
+    list_display = ("created_at", "id", "created_by", "token_redacted", "expires")
+    sort_fields = ("id", "created_at", "created_by", "expires")
+    readonly_fields = ("created_at", "modified_at")
+    search_fields = ("id", "created_by__username", "token")

-    list_filter = ('created_by',)
-    ordering = ['-created_at']
+    fieldsets = (
+        (
+            "Token",
+            {
+                "fields": ("token", "expires"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Owner",
+            {
+                "fields": ("created_by",),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": ("created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
+    )
+
+    list_filter = ("created_by",)
+    ordering = ["-created_at"]
    list_per_page = 100


-class CustomWebhookAdmin(WebhookAdmin, ABIDModelAdmin):
-    list_display = ('created_at', 'created_by', 'abid', *WebhookAdmin.list_display)
-    sort_fields = ('created_at', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
-    readonly_fields = ('created_at', 'modified_at', 'abid_info', *WebhookAdmin.readonly_fields)
+class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
+    list_display = ("created_at", "created_by", "id", *WebhookAdmin.list_display)
+    sort_fields = ("created_at", "created_by", "id", "referenced_model", "endpoint", "last_success", "last_error")
+    readonly_fields = ("created_at", "modified_at", *WebhookAdmin.readonly_fields)
+
+    fieldsets = (
+        (
+            "Webhook",
+            {
+                "fields": ("name", "signal", "referenced_model", "endpoint"),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Authentication",
+            {
+                "fields": ("auth_token",),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Status",
+            {
+                "fields": ("enabled", "last_success", "last_error"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Owner",
+            {
+                "fields": ("created_by",),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": ("created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
+    )
+
+    def lookup_allowed(self, lookup: str, value: str, request: HttpRequest | None = None) -> bool:
+        """Preserve WebhookAdmin's auth token filter with Django's current admin signature."""
+        return not lookup.startswith("auth_token") and admin.ModelAdmin.lookup_allowed(self, lookup, value, request)


-def register_admin(admin_site):
+def register_admin(admin_site: admin.AdminSite) -> None:
    admin_site.register(APIToken, APITokenAdmin)
    admin_site.register(get_webhook_model(), CustomWebhookAdmin)
@@ -1,15 +1,14 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"

 from django.apps import AppConfig

-import abx
-

 class APIConfig(AppConfig):
-    name = 'api'
+    name = "archivebox.api"
+    label = "api"


-@abx.hookimpl
 def register_admin(admin_site):
-    from api.admin import register_admin
+    from archivebox.api.admin import register_admin
+
    register_admin(admin_site)
@@ -1,21 +1,19 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"

-from typing import Any, Optional, cast
 from datetime import timedelta

-from django.http import HttpRequest
 from django.utils import timezone
-from django.contrib.auth import login
+from django.http import HttpRequest
 from django.contrib.auth import authenticate
-from django.contrib.auth.models import AbstractBaseUser
+from django.contrib.auth.models import User

-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth, django_auth_superuser
+from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth
 from ninja.errors import HttpError


-def get_or_create_api_token(user):
-    from api.models import APIToken
-    
+def get_or_create_api_token(user: User | None):
+    from archivebox.api.models import APIToken
+
    if user and user.is_superuser:
        api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
        if api_tokens.exists():
@@ -24,116 +22,122 @@ def get_or_create_api_token(user):
        else:
            # does not exist, create a new one
            api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
-        
+
+        if api_token is None:
+            return None
        assert api_token.is_valid(), f"API token is not valid {api_token}"

        return api_token
    return None


-def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
+def auth_using_token(token: str | None, request: HttpRequest | None = None) -> User | None:
    """Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
-    from api.models import APIToken        # lazy import model to avoid loading it at urls.py import time
-    
-    user = None
+    from archivebox.api.models import APIToken  # lazy import model to avoid loading it at urls.py import time

-    submitted_empty_form = str(token).strip() in ('string', '', 'None', 'null')
+    user: User | None = None
+
+    submitted_empty_form = str(token).strip() in ("string", "", "None", "null")
    if not submitted_empty_form:
        try:
-            token = APIToken.objects.get(token=token)
-            if token.is_valid():
-                user = token.created_by
-                request._api_token = token
+            api_token = APIToken.objects.get(token=token)
+            if api_token.is_valid() and isinstance(api_token.created_by, User):
+                user = api_token.created_by
+                if request is not None:
+                    setattr(request, "_api_token", api_token)
        except APIToken.DoesNotExist:
            pass

-    if not user:
-        # print('[❌] Failed to authenticate API user using API Key:', request)
-        return None
-    
-    return cast(AbstractBaseUser, user)
+    return user

-def auth_using_password(username, password, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
+
+def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
    """Given a username and password, check if they are valid and return the corresponding user"""
-    user = None
-    
-    submitted_empty_form = (username, password) in (('string', 'string'), ('', ''), (None, None))
+    user: User | None = None
+
+    submitted_empty_form = (username, password) in (("string", "string"), ("", ""), (None, None))
    if not submitted_empty_form:
-        user = authenticate(
+        authenticated_user = authenticate(
            username=username,
            password=password,
        )
-
-    if not user:
-        # print('[❌] Failed to authenticate API user using API Key:', request)
-        user = None
-
-    return cast(AbstractBaseUser | None, user)
+        if isinstance(authenticated_user, User):
+            user = authenticated_user
+    return user


 ### Base Auth Types


-class APITokenAuthCheck:
-    """The base class for authentication methods that use an api.models.APIToken"""
-    def authenticate(self, request: HttpRequest, key: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        request.user = auth_using_token(
-            token=key,
-            request=request,
-        )
-        if request.user and request.user.pk:
-            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
-            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
-            request._api_auth_method = self.__class__.__name__
-
-            if not request.user.is_superuser:
-                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
-        return request.user
-
-
-class UserPassAuthCheck:
-    """The base class for authentication methods that use a username & password"""
-    def authenticate(self, request: HttpRequest, username: Optional[str]=None, password: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        request.user = auth_using_password(
-            username=username,
-            password=password,
-            request=request,
-        )
-        if request.user and request.user.pk:
-            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
-            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
-            request._api_auth_method = self.__class__.__name__
-
-            if not request.user.is_superuser:
-                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
-
-        return request.user
+def _require_superuser(user: User | None, request: HttpRequest, auth_method: str) -> User | None:
+    if user and user.pk:
+        request.user = user
+        setattr(request, "_api_auth_method", auth_method)
+        if not user.is_superuser:
+            raise HttpError(403, "Valid credentials but User does not have permission (make sure user.is_superuser=True)")
+    return user


 ### Django-Ninja-Provided Auth Methods

-class HeaderTokenAuth(APITokenAuthCheck, APIKeyHeader):
+
+class HeaderTokenAuth(APIKeyHeader):
    """Allow authenticating by passing X-API-Key=xyz as a request header"""
+
    param_name = "X-ArchiveBox-API-Key"

-class BearerTokenAuth(APITokenAuthCheck, HttpBearer):
-    """Allow authenticating by passing Bearer=xyz as a request header"""
-    pass
+    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
+        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)

-class QueryParamTokenAuth(APITokenAuthCheck, APIKeyQuery):
+
+class BearerTokenAuth(HttpBearer):
+    """Allow authenticating by passing Bearer=xyz as a request header"""
+
+    def authenticate(self, request: HttpRequest, token: str) -> User | None:
+        return _require_superuser(auth_using_token(token=token, request=request), request, self.__class__.__name__)
+
+
+class QueryParamTokenAuth(APIKeyQuery):
    """Allow authenticating by passing api_key=xyz as a GET/POST query parameter"""
+
    param_name = "api_key"

-class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
+    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
+        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
+
+
+class UsernameAndPasswordAuth(HttpBasicAuth):
    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
-    pass
+
+    def authenticate(self, request: HttpRequest, username: str, password: str) -> User | None:
+        return _require_superuser(
+            auth_using_password(username=username, password=password, request=request),
+            request,
+            self.__class__.__name__,
+        )
+
+
+class DjangoSessionAuth:
+    """Allow authenticating with existing Django session cookies (same-origin only)."""
+
+    def __call__(self, request: HttpRequest) -> User | None:
+        return self.authenticate(request)
+
+    def authenticate(self, request: HttpRequest, **kwargs) -> User | None:
+        user = getattr(request, "user", None)
+        if isinstance(user, User) and user.is_authenticated:
+            setattr(request, "_api_auth_method", self.__class__.__name__)
+            if not user.is_superuser:
+                raise HttpError(403, "Valid session but User does not have permission (make sure user.is_superuser=True)")
+            return user
+        return None
+

 ### Enabled Auth Methods

 API_AUTH_METHODS = [
    HeaderTokenAuth(),
    BearerTokenAuth(),
-    QueryParamTokenAuth(), 
+    QueryParamTokenAuth(),
    # django_auth_superuser,       # django admin cookie auth, not secure to use with csrf=False
-    UsernameAndPasswordAuth(),
 ]
@@ -0,0 +1,32 @@
+__package__ = "archivebox.api"
+
+from django.http import HttpResponse
+
+
+class ApiCorsMiddleware:
+    """Attach permissive CORS headers for API routes (token-based auth)."""
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        if request.path.startswith("/api/"):
+            if request.method == "OPTIONS" and request.META.get("HTTP_ACCESS_CONTROL_REQUEST_METHOD"):
+                response = HttpResponse(status=204)
+                return self._add_cors_headers(request, response)
+
+            response = self.get_response(request)
+            return self._add_cors_headers(request, response)
+
+        return self.get_response(request)
+
+    def _add_cors_headers(self, request, response):
+        origin = request.META.get("HTTP_ORIGIN")
+        if not origin:
+            return response
+
+        response["Access-Control-Allow-Origin"] = "*"
+        response["Access-Control-Allow-Methods"] = "GET, POST, PUT, PATCH, DELETE, OPTIONS"
+        response["Access-Control-Allow-Headers"] = "Authorization, X-ArchiveBox-API-Key, Content-Type, X-CSRFToken"
+        response["Access-Control-Max-Age"] = "600"
+        return response
@@ -1,29 +1,239 @@
-# Generated by Django 4.2.11 on 2024-04-25 04:19
+# Generated by hand on 2025-12-29
+# Creates APIToken and OutboundWebhook tables using raw SQL

-import api.models
-from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
-import uuid
+import django.utils.timezone
+from django.conf import settings
+from archivebox.uuid_compat import uuid7
+from archivebox.base_models.models import get_or_create_system_user_pk
+import archivebox.api.models
+import signal_webhooks.fields
+import signal_webhooks.utils


 class Migration(migrations.Migration):
-
    initial = True

    dependencies = [
+        ("auth", "0012_alter_user_first_name_max_length"),
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
-        migrations.CreateModel(
-            name='APIToken',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('token', models.CharField(default=api.models.generate_secret_token, max_length=32, unique=True)),
-                ('created', models.DateTimeField(auto_now_add=True)),
-                ('expires', models.DateTimeField(blank=True, null=True)),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunSQL(
+                    sql="""
+                -- Create api_apitoken table
+                CREATE TABLE IF NOT EXISTS api_apitoken (
+                    id TEXT PRIMARY KEY NOT NULL,
+                    created_at DATETIME NOT NULL,
+                    modified_at DATETIME NOT NULL,
+
+                    token VARCHAR(32) NOT NULL UNIQUE,
+                    expires DATETIME,
+
+                    created_by_id INTEGER NOT NULL,
+
+                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
+                );
+                CREATE INDEX IF NOT EXISTS api_apitoken_created_by_id_idx ON api_apitoken(created_by_id);
+                CREATE INDEX IF NOT EXISTS api_apitoken_created_at_idx ON api_apitoken(created_at);
+                CREATE INDEX IF NOT EXISTS api_apitoken_token_idx ON api_apitoken(token);
+
+                -- Create api_outboundwebhook table
+                CREATE TABLE IF NOT EXISTS api_outboundwebhook (
+                    id TEXT PRIMARY KEY NOT NULL,
+                    created_at DATETIME NOT NULL,
+                    modified_at DATETIME NOT NULL,
+
+                    name VARCHAR(255) NOT NULL UNIQUE,
+                    signal VARCHAR(255) NOT NULL,
+                    ref VARCHAR(1024) NOT NULL,
+                    endpoint VARCHAR(2048) NOT NULL,
+                    headers TEXT NOT NULL DEFAULT '{}',
+                    auth_token TEXT NOT NULL DEFAULT '',
+                    enabled BOOLEAN NOT NULL DEFAULT 1,
+                    keep_last_response BOOLEAN NOT NULL DEFAULT 0,
+                    created DATETIME NOT NULL,
+                    updated DATETIME NOT NULL,
+                    last_response TEXT NOT NULL DEFAULT '',
+                    last_success DATETIME,
+                    last_failure DATETIME,
+
+                    created_by_id INTEGER NOT NULL,
+
+                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
+                );
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_by_id_idx ON api_outboundwebhook(created_by_id);
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
+                    """,
+                    reverse_sql="""
+                DROP TABLE IF EXISTS api_outboundwebhook;
+                DROP TABLE IF EXISTS api_apitoken;
+                    """,
+                ),
+            ],
+            state_operations=[
+                migrations.CreateModel(
+                    name="APIToken",
+                    fields=[
+                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
+                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
+                        ("modified_at", models.DateTimeField(auto_now=True)),
+                        ("token", models.CharField(default=archivebox.api.models.generate_secret_token, max_length=32, unique=True)),
+                        ("expires", models.DateTimeField(blank=True, null=True)),
+                        (
+                            "created_by",
+                            models.ForeignKey(
+                                default=get_or_create_system_user_pk,
+                                on_delete=django.db.models.deletion.CASCADE,
+                                to=settings.AUTH_USER_MODEL,
+                            ),
+                        ),
+                    ],
+                    options={
+                        "verbose_name": "API Key",
+                        "verbose_name_plural": "API Keys",
+                        "app_label": "api",
+                    },
+                ),
+                migrations.CreateModel(
+                    name="OutboundWebhook",
+                    fields=[
+                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
+                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
+                        ("modified_at", models.DateTimeField(auto_now=True)),
+                        (
+                            "name",
+                            models.CharField(db_index=True, help_text="Webhook name.", max_length=255, unique=True, verbose_name="name"),
+                        ),
+                        (
+                            "signal",
+                            models.CharField(
+                                choices=[
+                                    ("CREATE", "Create"),
+                                    ("UPDATE", "Update"),
+                                    ("DELETE", "Delete"),
+                                    ("M2M", "M2M changed"),
+                                    ("CREATE_OR_UPDATE", "Create or Update"),
+                                    ("CREATE_OR_DELETE", "Create or Delete"),
+                                    ("CREATE_OR_M2M", "Create or M2M changed"),
+                                    ("UPDATE_OR_DELETE", "Update or Delete"),
+                                    ("UPDATE_OR_M2M", "Update or M2M changed"),
+                                    ("DELETE_OR_M2M", "Delete or M2M changed"),
+                                    ("CREATE_UPDATE_OR_DELETE", "Create, Update or Delete"),
+                                    ("CREATE_UPDATE_OR_M2M", "Create, Update or M2M changed"),
+                                    ("CREATE_DELETE_OR_M2M", "Create, Delete or M2M changed"),
+                                    ("UPDATE_DELETE_OR_M2M", "Update, Delete or M2M changed"),
+                                    ("CREATE_UPDATE_DELETE_OR_M2M", "Create, Update or Delete, or M2M changed"),
+                                ],
+                                help_text="Signal the webhook fires to.",
+                                max_length=255,
+                                verbose_name="signal",
+                            ),
+                        ),
+                        (
+                            "ref",
+                            models.CharField(
+                                db_index=True,
+                                help_text="Dot import notation to the model the webhook is for.",
+                                max_length=1023,
+                                validators=[signal_webhooks.utils.model_from_reference],
+                                verbose_name="referenced model",
+                            ),
+                        ),
+                        (
+                            "endpoint",
+                            models.URLField(help_text="Target endpoint for this webhook.", max_length=2047, verbose_name="endpoint"),
+                        ),
+                        (
+                            "headers",
+                            models.JSONField(
+                                blank=True,
+                                default=dict,
+                                help_text="Headers to send with the webhook request.",
+                                validators=[signal_webhooks.utils.is_dict],
+                                verbose_name="headers",
+                            ),
+                        ),
+                        (
+                            "auth_token",
+                            signal_webhooks.fields.TokenField(
+                                blank=True,
+                                default="",
+                                help_text="Authentication token to use in an Authorization header.",
+                                max_length=8000,
+                                validators=[signal_webhooks.utils.decode_cipher_key],
+                                verbose_name="authentication token",
+                            ),
+                        ),
+                        ("enabled", models.BooleanField(default=True, help_text="Is this webhook enabled?", verbose_name="enabled")),
+                        (
+                            "keep_last_response",
+                            models.BooleanField(
+                                default=False,
+                                help_text="Should the webhook keep a log of the latest response it got?",
+                                verbose_name="keep last response",
+                            ),
+                        ),
+                        (
+                            "created",
+                            models.DateTimeField(auto_now_add=True, help_text="When the webhook was created.", verbose_name="created"),
+                        ),
+                        (
+                            "updated",
+                            models.DateTimeField(auto_now=True, help_text="When the webhook was last updated.", verbose_name="updated"),
+                        ),
+                        (
+                            "last_response",
+                            models.CharField(
+                                blank=True,
+                                default="",
+                                help_text="Latest response to this webhook.",
+                                max_length=8000,
+                                verbose_name="last response",
+                            ),
+                        ),
+                        (
+                            "last_success",
+                            models.DateTimeField(
+                                default=None,
+                                help_text="When the webhook last succeeded.",
+                                null=True,
+                                verbose_name="last success",
+                            ),
+                        ),
+                        (
+                            "last_failure",
+                            models.DateTimeField(
+                                default=None,
+                                help_text="When the webhook last failed.",
+                                null=True,
+                                verbose_name="last failure",
+                            ),
+                        ),
+                        (
+                            "created_by",
+                            models.ForeignKey(
+                                default=get_or_create_system_user_pk,
+                                on_delete=django.db.models.deletion.CASCADE,
+                                to=settings.AUTH_USER_MODEL,
+                            ),
+                        ),
+                    ],
+                    options={
+                        "verbose_name": "API Outbound Webhook",
+                        "app_label": "api",
+                    },
+                ),
+                migrations.AddConstraint(
+                    model_name="outboundwebhook",
+                    constraint=models.UniqueConstraint(fields=["ref", "endpoint"], name="prevent_duplicate_hooks_api_outboundwebhook"),
+                ),
            ],
        ),
    ]
@@ -1,17 +0,0 @@
-# Generated by Django 5.0.4 on 2024-04-26 05:28
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name='apitoken',
-            options={'verbose_name': 'API Key', 'verbose_name_plural': 'API Keys'},
-        ),
-    ]
@@ -1,77 +0,0 @@
-# Generated by Django 5.0.6 on 2024-06-03 01:52
-
-import abid_utils.models
-import charidfield.fields
-import django.db.models.deletion
-import signal_webhooks.fields
-import signal_webhooks.utils
-import uuid
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0002_alter_apitoken_options'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='apitoken',
-            old_name='user',
-            new_name='created_by',
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='abid',
-            field=charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='apt_', unique=True),
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='modified',
-            field=models.DateTimeField(auto_now=True),
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='uuid',
-            field=models.UUIDField(blank=True, null=True, unique=True),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False),
-        ),
-        migrations.CreateModel(
-            name='OutboundWebhook',
-            fields=[
-                ('name', models.CharField(db_index=True, help_text='Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).', max_length=255, unique=True, verbose_name='name')),
-                ('signal', models.CharField(choices=[('CREATE', 'Create'), ('UPDATE', 'Update'), ('DELETE', 'Delete'), ('M2M', 'M2M changed'), ('CREATE_OR_UPDATE', 'Create or Update'), ('CREATE_OR_DELETE', 'Create or Delete'), ('CREATE_OR_M2M', 'Create or M2M changed'), ('UPDATE_OR_DELETE', 'Update or Delete'), ('UPDATE_OR_M2M', 'Update or M2M changed'), ('DELETE_OR_M2M', 'Delete or M2M changed'), ('CREATE_UPDATE_OR_DELETE', 'Create, Update or Delete'), ('CREATE_UPDATE_OR_M2M', 'Create, Update or M2M changed'), ('CREATE_DELETE_OR_M2M', 'Create, Delete or M2M changed'), ('UPDATE_DELETE_OR_M2M', 'Update, Delete or M2M changed'), ('CREATE_UPDATE_DELETE_OR_M2M', 'Create, Update or Delete, or M2M changed')], help_text='The type of event the webhook should fire for (e.g. Create, Update, Delete).', max_length=255, verbose_name='signal')),
-                ('ref', models.CharField(db_index=True, help_text='Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).', max_length=1023, validators=[signal_webhooks.utils.model_from_reference], verbose_name='referenced model')),
-                ('endpoint', models.URLField(help_text='External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).', max_length=2047, verbose_name='endpoint')),
-                ('headers', models.JSONField(blank=True, default=dict, help_text='Headers to send with the webhook request.', validators=[signal_webhooks.utils.is_dict], verbose_name='headers')),
-                ('auth_token', signal_webhooks.fields.TokenField(blank=True, default='', help_text='Authentication token to use in an Authorization header.', max_length=8000, validators=[signal_webhooks.utils.decode_cipher_key], verbose_name='authentication token')),
-                ('enabled', models.BooleanField(default=True, help_text='Is this webhook enabled?', verbose_name='enabled')),
-                ('keep_last_response', models.BooleanField(default=False, help_text='Should the webhook keep a log of the latest response it got?', verbose_name='keep last response')),
-                ('updated', models.DateTimeField(auto_now=True, help_text='When the webhook was last updated.', verbose_name='updated')),
-                ('last_response', models.CharField(blank=True, default='', help_text='Latest response to this webhook.', max_length=8000, verbose_name='last response')),
-                ('last_success', models.DateTimeField(default=None, help_text='When the webhook last succeeded.', null=True, verbose_name='last success')),
-                ('last_failure', models.DateTimeField(default=None, help_text='When the webhook last failed.', null=True, verbose_name='last failure')),
-                ('created', models.DateTimeField(auto_now_add=True)),
-                ('modified', models.DateTimeField(auto_now=True)),
-                ('id', models.UUIDField(blank=True, null=True, unique=True)),
-                ('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
-                ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='whk_', unique=True)),
-                ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-            options={
-                'verbose_name': 'API Outbound Webhook',
-                'abstract': False,
-            },
-        ),
-        migrations.AddConstraint(
-            model_name='outboundwebhook',
-            constraint=models.UniqueConstraint(fields=('ref', 'endpoint'), name='prevent_duplicate_hooks_api_outboundwebhook'),
-        ),
-    ]
@@ -1,24 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 10:44
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0003_rename_user_apitoken_created_by_apitoken_abid_and_more'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='uuid',
-            field=models.UUIDField(blank=True, editable=False, null=True, unique=True),
-        ),
-    ]
@@ -1,22 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:40
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0004_alter_apitoken_id_alter_apitoken_uuid'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='apitoken',
-            name='uuid',
-        ),
-        migrations.RemoveField(
-            model_name='outboundwebhook',
-            name='id',
-        ),
-    ]
@@ -1,29 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:43
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more'),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='outboundwebhook',
-            old_name='uuid',
-            new_name='id'
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-    ]
@@ -1,22 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:52
-
-import abid_utils.models
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0006_remove_outboundwebhook_uuid_apitoken_id_and_more'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created_by',
-            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-    ]
@@ -1,47 +0,0 @@
-# Generated by Django 5.1 on 2024-09-04 23:32
-
-import abid_utils.models
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0007_alter_apitoken_created_by'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created',
-            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created_by',
-            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='created',
-            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='created_by',
-            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='id',
-            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
-        ),
-    ]
@@ -1,39 +0,0 @@
-# Generated by Django 5.1 on 2024-09-05 00:26
-
-import abid_utils.models
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0008_alter_apitoken_created_alter_apitoken_created_by_and_more'),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='apitoken',
-            old_name='created',
-            new_name='created_at',
-        ),
-        migrations.RenameField(
-            model_name='apitoken',
-            old_name='modified',
-            new_name='modified_at',
-        ),
-        migrations.RenameField(
-            model_name='outboundwebhook',
-            old_name='modified',
-            new_name='modified_at',
-        ),
-        migrations.AddField(
-            model_name='outboundwebhook',
-            name='created_at',
-            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='created',
-            field=models.DateTimeField(auto_now_add=True, help_text='When the webhook was created.', verbose_name='created'),
-        ),
-    ]
@@ -1,127 +1,54 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"

 import secrets
-from datetime import timedelta
+from archivebox.uuid_compat import uuid7

 from django.conf import settings
 from django.db import models
 from django.utils import timezone
-
+from django_stubs_ext.db.models import TypedModelMeta
 from signal_webhooks.models import WebhookBase

-from django_stubs_ext.db.models import TypedModelMeta
-
-from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
-
+from archivebox.base_models.models import get_or_create_system_user_pk


 def generate_secret_token() -> str:
-    # returns cryptographically secure string with len() == 32
    return secrets.token_hex(16)


-class APIToken(ABIDModel):
-    """
-    A secret key generated by a User that's used to authenticate REST API requests to ArchiveBox.
-    """
-    # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
-    abid_prefix = 'apt_'
-    abid_ts_src = 'self.created_at'
-    abid_uri_src = 'self.created_by_id'
-    abid_subtype_src = '"01"'
-    abid_rand_src = 'self.id'
-    abid_drift_allowed = True
-
-    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
-    abid = ABIDField(prefix=abid_prefix)
-
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+class APIToken(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
+    created_at = models.DateTimeField(default=timezone.now, db_index=True)
    modified_at = models.DateTimeField(auto_now=True)
-
    token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
    expires = models.DateTimeField(null=True, blank=True)

    class Meta(TypedModelMeta):
+        app_label = "api"
        verbose_name = "API Key"
        verbose_name_plural = "API Keys"

    def __str__(self) -> str:
        return self.token

-    def __repr__(self) -> str:
-        return f'<APIToken user={self.created_by.username} token={self.token_redacted}>'
-
-    def __json__(self) -> dict:
-        return {
-            "TYPE":             "APIToken",    
-            "id":               str(self.pk),
-            "abid":             str(self.ABID),
-            "created_by_id":    str(self.created_by_id),
-            "token":            self.token,
-            "created_at":       self.created_at.isoformat(),
-            "expires":          self.expires_as_iso8601,
-        }
-
-    @property
-    def expires_as_iso8601(self):
-        """Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
-        expiry_date = self.expires or (timezone.now() + timedelta(days=365 * 100))
-
-        return expiry_date.isoformat()
-    
    @property
    def token_redacted(self):
-        return f'************{self.token[-4:]}'
+        return f"************{self.token[-4:]}"

    def is_valid(self, for_date=None):
-        for_date = for_date or timezone.now()
-
-        if self.expires and self.expires < for_date:
-            return False
-
-        return True
+        return not self.expires or self.expires >= (for_date or timezone.now())


-
-
-
-
-# monkey patch django-signals-webhooks to change how it shows up in Admin UI
-
-class OutboundWebhook(ABIDModel, WebhookBase):
-    """
-    Model used in place of (extending) signals_webhooks.models.WebhookModel. Swapped using:
-        settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
-    """
-    abid_prefix = 'whk_'
-    abid_ts_src = 'self.created_at'
-    abid_uri_src = 'self.endpoint'
-    abid_subtype_src = 'self.ref'
-    abid_rand_src = 'self.id'
-    abid_drift_allowed = True
-
-    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
-    abid = ABIDField(prefix=abid_prefix)
-
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+class OutboundWebhook(WebhookBase):
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
+    created_at = models.DateTimeField(default=timezone.now, db_index=True)
    modified_at = models.DateTimeField(auto_now=True)

-    # More fields here: WebhookBase...
-
-    WebhookBase._meta.get_field('name').help_text = (
-        'Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).')
-    WebhookBase._meta.get_field('signal').help_text = (
-        'The type of event the webhook should fire for (e.g. Create, Update, Delete).')
-    WebhookBase._meta.get_field('ref').help_text = (
-        'Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).')
-    WebhookBase._meta.get_field('endpoint').help_text = (
-        'External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).')
-
    class Meta(WebhookBase.Meta):
-        verbose_name = 'API Outbound Webhook'
-
+        app_label = "api"
+        verbose_name = "API Outbound Webhook"

    def __str__(self) -> str:
-        return f'[{self.abid}] {self.ref} -> {self.endpoint}'
+        return f"[{self.id}] {self.ref} -> {self.endpoint}"
@@ -1,30 +0,0 @@
-__package__ = 'archivebox.api'
-
-# from django.test import TestCase
-# from ninja.testing import TestClient
-
-# from .routes_cli import router
-
-# class ArchiveBoxCLIAPITestCase(TestCase):
-#     def setUp(self):
-#         self.client = TestClient(router)
-
-#     def test_add_endpoint(self):
-#         response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
-
-#     def test_remove_endpoint(self):
-#         response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
-
-#     def test_update_endpoint(self):
-#         response = self.client.post("/update", json={})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
-
-#     def test_list_all_endpoint(self):
-#         response = self.client.post("/list_all", json={})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
@@ -1,4 +1,4 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"

 from django.urls import path
 from django.views.generic.base import RedirectView
@@ -6,11 +6,10 @@ from django.views.generic.base import RedirectView
 from .v1_api import urls as v1_api_urls

 urlpatterns = [
-    path("",                 RedirectView.as_view(url='/api/v1')),
-
-    path("v1/",              v1_api_urls),
-    path("v1",               RedirectView.as_view(url='/api/v1/docs')),
-
+    path("", RedirectView.as_view(url="/api/v1/docs")),
+    path("v1/", RedirectView.as_view(url="/api/v1/docs")),
+    path("v1/", v1_api_urls),
+    path("v1", RedirectView.as_view(url="/api/v1/docs")),
    # ... v2 can be added here ...
    # path("v2/",              v2_api_urls),
    # path("v2",               RedirectView.as_view(url='/api/v2/docs')),
@@ -1,4 +1,4 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"


 from io import StringIO
@@ -7,6 +7,7 @@ from contextlib import redirect_stdout, redirect_stderr

 from django.http import HttpRequest, HttpResponse
 from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, PermissionDenied
+from django.contrib.auth.models import User

 from ninja import NinjaAPI, Swagger

@@ -15,12 +16,13 @@ from ninja import NinjaAPI, Swagger
 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH

-from api.auth import API_AUTH_METHODS
+from archivebox.api.auth import API_AUTH_METHODS
+from archivebox.api.models import APIToken


-COMMIT_HASH = get_COMMIT_HASH() or 'unknown'
+COMMIT_HASH = get_COMMIT_HASH() or "unknown"

-html_description=f'''
+html_description = f"""
 <h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
 <br/>
 <i><b>WARNING: This API is still in an early development stage and may change!</b></i>
@@ -33,44 +35,47 @@ html_description=f'''
 <li>📜 See the API source code: <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api"><code>archivebox/api/</code></a></li>
 </ul>
 <small>Served by ArchiveBox v{VERSION} (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{COMMIT_HASH}"><code>{COMMIT_HASH[:8]}</code></a>), API powered by <a href="https://django-ninja.dev/"><code>django-ninja</code></a>.</small>
-'''
+"""


 def register_urls(api: NinjaAPI) -> NinjaAPI:
-    api.add_router('/auth/',     'api.v1_auth.router')
-    api.add_router('/core/',     'api.v1_core.router')
-    api.add_router('/cli/',      'api.v1_cli.router')
+    api.add_router("/auth/", "archivebox.api.v1_auth.router")
+    api.add_router("/core/", "archivebox.api.v1_core.router")
+    api.add_router("/crawls/", "archivebox.api.v1_crawls.router")
+    api.add_router("/cli/", "archivebox.api.v1_cli.router")
+    api.add_router("/machine/", "archivebox.api.v1_machine.router")
    return api


-class NinjaAPIWithIOCapture(NinjaAPI):    
+class NinjaAPIWithIOCapture(NinjaAPI):
    def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
        stdout, stderr = StringIO(), StringIO()

        with redirect_stderr(stderr):
            with redirect_stdout(stdout):
-                request.stdout = stdout
-                request.stderr = stderr
+                setattr(request, "stdout", stdout)
+                setattr(request, "stderr", stderr)

                response = super().create_temporal_response(request)

-        # Diable caching of API responses entirely
-        response['Cache-Control'] = 'no-store'
+        # Disable caching of API responses entirely
+        response["Cache-Control"] = "no-store"

        # Add debug stdout and stderr headers to response
-        response['X-ArchiveBox-Stdout'] = str(request.stdout)[200:]
-        response['X-ArchiveBox-Stderr'] = str(request.stderr)[200:]
+        response["X-ArchiveBox-Stdout"] = stdout.getvalue().replace("\n", "\\n")[:200]
+        response["X-ArchiveBox-Stderr"] = stderr.getvalue().replace("\n", "\\n")[:200]
        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'

        # Add Auth Headers to response
-        api_token = getattr(request, '_api_token', None)
-        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else 'Never'
+        api_token_attr = getattr(request, "_api_token", None)
+        api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
+        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"

-        response['X-ArchiveBox-Auth-Method'] = getattr(request, '_api_auth_method', None) or 'None'
-        response['X-ArchiveBox-Auth-Expires'] = token_expiry
-        response['X-ArchiveBox-Auth-Token-Id'] = api_token.abid if api_token else 'None'
-        response['X-ArchiveBox-Auth-User-Id'] = request.user.pk if request.user.pk else 'None'
-        response['X-ArchiveBox-Auth-User-Username'] = request.user.username if request.user.pk else 'None'
+        response["X-ArchiveBox-Auth-Method"] = str(getattr(request, "_api_auth_method", "None"))
+        response["X-ArchiveBox-Auth-Expires"] = token_expiry
+        response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
+        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if getattr(request.user, "pk", None) else "None"
+        response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"

        # import ipdb; ipdb.set_trace()
        # print('RESPONDING NOW', response)
@@ -79,10 +84,9 @@ class NinjaAPIWithIOCapture(NinjaAPI):


 api = NinjaAPIWithIOCapture(
-    title='ArchiveBox API',
+    title="ArchiveBox API",
    description=html_description,
-    version='1.0.0',
-    csrf=False,
+    version=VERSION,
    auth=API_AUTH_METHODS,
    urls_namespace="api-1",
    docs=Swagger(settings={"persistAuthorization": True}),
@@ -99,15 +103,15 @@ def generic_exception_handler(request, err):
    if isinstance(err, (ObjectDoesNotExist, EmptyResultSet, PermissionDenied)):
        status = 404

-    print(''.join(format_exception(err)))
+    print("".join(format_exception(err)))

    return api.create_response(
        request,
        {
            "succeeded": False,
-            "message": f'{err.__class__.__name__}: {err}',
+            "message": f"{err.__class__.__name__}: {err}",
            "errors": [
-                ''.join(format_exception(err)),
+                "".join(format_exception(err)),
                # or send simpler parent-only traceback:
                # *([str(err.__context__)] if getattr(err, '__context__', None) else []),
            ],
@@ -116,7 +120,6 @@ def generic_exception_handler(request, err):
    )


-
 # import orjson
 # from ninja.renderers import BaseRenderer
 # class ORJSONRenderer(BaseRenderer):
@@ -1,26 +1,28 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"

-from typing import Optional
+from django.http import HttpRequest

 from ninja import Router, Schema
-from django.utils import timezone
-from datetime import timedelta

-from api.models import APIToken
-from api.auth import auth_using_token, auth_using_password, get_or_create_api_token
+from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token


-router = Router(tags=['Authentication'], auth=None)
+router = Router(tags=["Authentication"], auth=None)


 class PasswordAuthSchema(Schema):
    """Schema for a /get_api_token request"""
-    username: Optional[str] = None
-    password: Optional[str] = None
+
+    username: str | None = None
+    password: str | None = None


-@router.post("/get_api_token", auth=None, summary='Generate an API token for a given username & password (or currently logged-in user)')             # auth=None because they are not authed yet
-def get_api_token(request, auth_data: PasswordAuthSchema):
+@router.post(
+    "/get_api_token",
+    auth=None,
+    summary="Generate an API token for a given username & password (or currently logged-in user)",
+)  # auth=None because they are not authed yet
+def get_api_token(request: HttpRequest, auth_data: PasswordAuthSchema):
    user = auth_using_password(
        username=auth_data.username,
        password=auth_data.password,
@@ -30,24 +32,34 @@ def get_api_token(request, auth_data: PasswordAuthSchema):
    if user and user.is_superuser:
        api_token = get_or_create_api_token(user)
        assert api_token is not None, "Failed to create API token"
-        return api_token.__json__()
-    
-    return {"success": False, "errors": ["Invalid credentials"]}
+        return {
+            "success": True,
+            "user_id": str(user.pk),
+            "username": user.username,
+            "token": api_token.token,
+            "expires": api_token.expires.isoformat() if api_token.expires else None,
+        }

+    return {"success": False, "errors": ["Invalid credentials"]}


 class TokenAuthSchema(Schema):
    """Schema for a /check_api_token request"""
+
    token: str


-@router.post("/check_api_token", auth=None, summary='Validate an API token to make sure its valid and non-expired')        # auth=None because they are not authed yet
-def check_api_token(request, token_data: TokenAuthSchema):
+@router.post(
+    "/check_api_token",
+    auth=None,
+    summary="Validate an API token to make sure its valid and non-expired",
+)  # auth=None because they are not authed yet
+def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
    user = auth_using_token(
        token=token_data.token,
        request=request,
    )
    if user:
        return {"success": True, "user_id": str(user.pk)}
-    
+
    return {"success": False, "user_id": None}
@@ -1,169 +1,189 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"

-from typing import List, Dict, Any, Optional
+import json
+from io import StringIO
+from typing import Any
 from enum import Enum

+from django.http import HttpRequest
+
 from ninja import Router, Schema

-from ..main import (
-    add,
-    remove,
-    update,
-    list_all,
-    schedule,
-)
 from archivebox.misc.util import ansi_to_html
 from archivebox.config.common import ARCHIVING_CONFIG


-from .auth import API_AUTH_METHODS
+# from .auth import API_AUTH_METHODS

 # router for API that exposes archivebox cli subcommands as REST endpoints
-router = Router(tags=['ArchiveBox CLI Sub-Commands'], auth=API_AUTH_METHODS)
+router = Router(tags=["ArchiveBox CLI Sub-Commands"])


 # Schemas

-JSONType = List[Any] | Dict[str, Any] | bool | int | str | None
+JSONType = list[Any] | dict[str, Any] | bool | int | str | None
+

 class CLICommandResponseSchema(Schema):
    success: bool
-    errors: List[str]
+    errors: list[str]
    result: JSONType
+    result_format: str = "str"
    stdout: str
    stderr: str

+
 class FilterTypeChoices(str, Enum):
-    exact = 'exact'
-    substring = 'substring'
-    regex = 'regex'
-    domain = 'domain'
-    tag = 'tag'
-    timestamp = 'timestamp'
+    exact = "exact"
+    substring = "substring"
+    regex = "regex"
+    domain = "domain"
+    tag = "tag"
+    timestamp = "timestamp"
+

 class StatusChoices(str, Enum):
-    indexed = 'indexed'
-    archived = 'archived'
-    unarchived = 'unarchived'
-    present = 'present'
-    valid = 'valid'
-    invalid = 'invalid'
-    duplicate = 'duplicate'
-    orphaned = 'orphaned'
-    corrupted = 'corrupted'
-    unrecognized = 'unrecognized'
+    indexed = "indexed"
+    archived = "archived"
+    unarchived = "unarchived"
+    present = "present"
+    valid = "valid"
+    invalid = "invalid"
+    duplicate = "duplicate"
+    orphaned = "orphaned"
+    corrupted = "corrupted"
+    unrecognized = "unrecognized"


 class AddCommandSchema(Schema):
-    urls: List[str]
+    urls: list[str]
    tag: str = ""
    depth: int = 0
-    update: bool = not ARCHIVING_CONFIG.ONLY_NEW  # Default to the opposite of ARCHIVING_CONFIG.ONLY_NEW
-    update_all: bool = False
-    index_only: bool = False
-    overwrite: bool = False
-    init: bool = False
-    extractors: str = ""
    parser: str = "auto"
+    plugins: str = ""
+    update: bool = not ARCHIVING_CONFIG.ONLY_NEW  # Default to the opposite of ARCHIVING_CONFIG.ONLY_NEW
+    overwrite: bool = False
+    index_only: bool = False
+

 class UpdateCommandSchema(Schema):
-    resume: Optional[float] = 0
-    only_new: bool = ARCHIVING_CONFIG.ONLY_NEW
-    index_only: bool = False
-    overwrite: bool = False
-    after: Optional[float] = 0
-    before: Optional[float] = 999999999999999
-    status: Optional[StatusChoices] = StatusChoices.unarchived
-    filter_type: Optional[str] = FilterTypeChoices.substring
-    filter_patterns: Optional[List[str]] = ['https://example.com']
-    extractors: Optional[str] = ""
+    resume: str | None = None
+    after: float | None = 0
+    before: float | None = 999999999999999
+    filter_type: str | None = FilterTypeChoices.substring
+    filter_patterns: list[str] | None = ["https://example.com"]
+    batch_size: int = 100
+    continuous: bool = False
+

 class ScheduleCommandSchema(Schema):
-    import_path: Optional[str] = None
+    import_path: str | None = None
    add: bool = False
-    every: Optional[str] = None
-    tag: str = ''
+    show: bool = False
+    foreground: bool = False
+    run_all: bool = False
+    quiet: bool = False
+    every: str | None = None
+    tag: str = ""
    depth: int = 0
    overwrite: bool = False
    update: bool = not ARCHIVING_CONFIG.ONLY_NEW
    clear: bool = False

+
 class ListCommandSchema(Schema):
-    filter_patterns: Optional[List[str]] = ['https://example.com']
+    filter_patterns: list[str] | None = ["https://example.com"]
    filter_type: str = FilterTypeChoices.substring
-    status: Optional[StatusChoices] = StatusChoices.indexed
-    after: Optional[float] = 0
-    before: Optional[float] = 999999999999999
-    sort: str = 'bookmarked_at'
+    status: StatusChoices = StatusChoices.indexed
+    after: float | None = 0
+    before: float | None = 999999999999999
+    sort: str = "bookmarked_at"
    as_json: bool = True
    as_html: bool = False
-    as_csv: str | bool = 'timestamp,url'
+    as_csv: str | None = "timestamp,url"
    with_headers: bool = False

+
 class RemoveCommandSchema(Schema):
    delete: bool = True
-    after: Optional[float] = 0
-    before: Optional[float] = 999999999999999
+    after: float | None = 0
+    before: float | None = 999999999999999
    filter_type: str = FilterTypeChoices.exact
-    filter_patterns: Optional[List[str]] = ['https://example.com']
+    filter_patterns: list[str] | None = ["https://example.com"]


+@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
+def cli_add(request: HttpRequest, args: AddCommandSchema):
+    from archivebox.cli.archivebox_add import add

-
-
-@router.post("/add", response=CLICommandResponseSchema, summary='archivebox add [args] [urls]')
-def cli_add(request, args: AddCommandSchema):
-    result = add(
+    crawl, snapshots = add(
        urls=args.urls,
        tag=args.tag,
        depth=args.depth,
        update=args.update,
-        update_all=args.update_all,
        index_only=args.index_only,
        overwrite=args.overwrite,
-        init=args.init,
-        extractors=args.extractors,
+        plugins=args.plugins,
        parser=args.parser,
+        bg=True,  # Always run in background for API calls
+        created_by_id=request.user.pk,
    )

+    snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
+    result_payload = {
+        "crawl_id": str(crawl.id),
+        "num_snapshots": len(snapshot_ids),
+        "snapshot_ids": snapshot_ids,
+        "queued_urls": args.urls,
+    }
+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
+
    return {
        "success": True,
        "errors": [],
-        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+        "result": result_payload,
+        "result_format": "json",
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
    }


-@router.post("/update", response=CLICommandResponseSchema, summary='archivebox update [args] [filter_patterns]')
-def cli_update(request, args: UpdateCommandSchema):
+@router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
+def cli_update(request: HttpRequest, args: UpdateCommandSchema):
+    from archivebox.cli.archivebox_update import update
+
    result = update(
-        resume=args.resume,
-        only_new=args.only_new,
-        index_only=args.index_only,
-        overwrite=args.overwrite,
-        before=args.before,
+        filter_patterns=args.filter_patterns or [],
+        filter_type=args.filter_type or FilterTypeChoices.substring,
        after=args.after,
-        status=args.status,
-        filter_type=args.filter_type,
-        filter_patterns=args.filter_patterns,
-        extractors=args.extractors,
+        before=args.before,
+        resume=args.resume,
+        batch_size=args.batch_size,
+        continuous=args.continuous,
    )
+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
    return {
        "success": True,
        "errors": [],
        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
    }


-@router.post("/schedule", response=CLICommandResponseSchema, summary='archivebox schedule [args] [import_path]')
-def cli_schedule(request, args: ScheduleCommandSchema):
+@router.post("/schedule", response=CLICommandResponseSchema, summary="archivebox schedule [args] [import_path]")
+def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
+    from archivebox.cli.archivebox_schedule import schedule
+
    result = schedule(
        import_path=args.import_path,
        add=args.add,
        show=args.show,
+        foreground=args.foreground,
+        run_all=args.run_all,
+        quiet=args.quiet,
        clear=args.clear,
        every=args.every,
        tag=args.tag,
@@ -172,19 +192,23 @@ def cli_schedule(request, args: ScheduleCommandSchema):
        update=args.update,
    )

+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
    return {
        "success": True,
        "errors": [],
        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+        "result_format": "json",
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
    }


+@router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
+def cli_search(request: HttpRequest, args: ListCommandSchema):
+    from archivebox.cli.archivebox_search import search

-@router.post("/list", response=CLICommandResponseSchema, summary='archivebox list [args] [filter_patterns]')
-def cli_list(request, args: ListCommandSchema):
-    result = list_all(
+    result = search(
        filter_patterns=args.filter_patterns,
        filter_type=args.filter_type,
        status=args.status,
@@ -197,40 +221,64 @@ def cli_list(request, args: ListCommandSchema):
        with_headers=args.with_headers,
    )

-    result_format = 'txt'
+    result_format = "txt"
    if args.as_json:
        result_format = "json"
+        result = json.loads(result)
    elif args.as_html:
        result_format = "html"
    elif args.as_csv:
        result_format = "csv"

+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
    return {
        "success": True,
        "errors": [],
        "result": result,
        "result_format": result_format,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
    }
-    


-@router.post("/remove", response=CLICommandResponseSchema, summary='archivebox remove [args] [filter_patterns]')
-def cli_remove(request, args: RemoveCommandSchema):
-    result = remove(
-        yes=True,            # no way to interactively ask for confirmation via API, so we force yes
+@router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
+def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
+    from archivebox.cli.archivebox_remove import remove
+    from archivebox.cli.archivebox_search import get_snapshots
+    from archivebox.core.models import Snapshot
+
+    filter_patterns = args.filter_patterns or []
+    snapshots_to_remove = get_snapshots(
+        filter_patterns=filter_patterns,
+        filter_type=args.filter_type,
+        after=args.after,
+        before=args.before,
+    )
+    removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)]
+
+    remove(
+        yes=True,  # no way to interactively ask for confirmation via API, so we force yes
        delete=args.delete,
+        snapshots=snapshots_to_remove,
        before=args.before,
        after=args.after,
        filter_type=args.filter_type,
-        filter_patterns=args.filter_patterns,
+        filter_patterns=filter_patterns,
    )
+
+    result = {
+        "removed_count": len(removed_snapshot_ids),
+        "removed_snapshot_ids": removed_snapshot_ids,
+        "remaining_snapshots": Snapshot.objects.count(),
+    }
+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
    return {
        "success": True,
        "errors": [],
        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+        "result_format": "json",
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
    }
-    
@@ -0,0 +1,201 @@
+__package__ = "archivebox.api"
+
+from uuid import UUID
+from datetime import datetime
+from django.http import HttpRequest
+from django.utils import timezone
+
+from django.contrib.auth import get_user_model
+from django.contrib.auth.models import User
+
+from ninja import Router, Schema
+from ninja.errors import HttpError
+
+from archivebox.core.models import Snapshot
+from archivebox.crawls.models import Crawl
+
+from .auth import API_AUTH_METHODS
+
+router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
+
+
+class CrawlSchema(Schema):
+    TYPE: str = "crawls.models.Crawl"
+
+    id: UUID
+
+    modified_at: datetime
+    created_at: datetime
+    created_by_id: str
+    created_by_username: str
+
+    status: str
+    retry_at: datetime | None
+
+    urls: str
+    max_depth: int
+    max_urls: int
+    max_size: int
+    tags_str: str
+    config: dict
+
+    # snapshots: List[SnapshotSchema]
+
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by_id)
+
+    @staticmethod
+    def resolve_created_by_username(obj):
+        user_model = get_user_model()
+        user = user_model.objects.get(id=obj.created_by_id)
+        username = getattr(user, "username", None)
+        return username if isinstance(username, str) else str(user)
+
+    @staticmethod
+    def resolve_snapshots(obj, context):
+        if bool(getattr(context["request"], "with_snapshots", False)):
+            return obj.snapshot_set.all().distinct()
+        return Snapshot.objects.none()
+
+
+class CrawlUpdateSchema(Schema):
+    status: str | None = None
+    retry_at: datetime | None = None
+    tags: list[str] | None = None
+    tags_str: str | None = None
+
+
+class CrawlCreateSchema(Schema):
+    urls: list[str]
+    max_depth: int = 0
+    max_urls: int = 0
+    max_size: int = 0
+    tags: list[str] | None = None
+    tags_str: str = ""
+    label: str = ""
+    notes: str = ""
+    config: dict = {}
+
+
+class CrawlDeleteResponseSchema(Schema):
+    success: bool
+    crawl_id: str
+    deleted_count: int
+    deleted_snapshots: int
+
+
+def normalize_tag_list(tags: list[str] | None = None, tags_str: str = "") -> list[str]:
+    if tags is not None:
+        return [tag.strip() for tag in tags if tag and tag.strip()]
+    return [tag.strip() for tag in tags_str.split(",") if tag.strip()]
+
+
+@router.get("/crawls", response=list[CrawlSchema], url_name="get_crawls")
+def get_crawls(request: HttpRequest):
+    return Crawl.objects.all().distinct()
+
+
+@router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
+def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
+    urls = [url.strip() for url in data.urls if url and url.strip()]
+    if not urls:
+        raise HttpError(400, "At least one URL is required")
+    if data.max_depth not in (0, 1, 2, 3, 4):
+        raise HttpError(400, "max_depth must be between 0 and 4")
+    if data.max_urls < 0:
+        raise HttpError(400, "max_urls must be >= 0")
+    if data.max_size < 0:
+        raise HttpError(400, "max_size must be >= 0")
+
+    tags = normalize_tag_list(data.tags, data.tags_str)
+    crawl = Crawl.objects.create(
+        urls="\n".join(urls),
+        max_depth=data.max_depth,
+        max_urls=data.max_urls,
+        max_size=data.max_size,
+        tags_str=",".join(tags),
+        label=data.label,
+        notes=data.notes,
+        config=data.config,
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at=timezone.now(),
+        created_by=request.user if isinstance(request.user, User) else None,
+    )
+    crawl.create_snapshots_from_urls()
+    return crawl
+
+
+@router.get("/crawl/{crawl_id}", response=CrawlSchema | str, url_name="get_crawl")
+def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
+    """Get a specific Crawl by id."""
+    setattr(request, "with_snapshots", with_snapshots)
+    setattr(request, "with_archiveresults", with_archiveresults)
+    crawl = Crawl.objects.get(id__icontains=crawl_id)
+
+    if crawl and as_rss:
+        # return snapshots as XML rss feed
+        urls = [
+            {"url": snapshot.url, "title": snapshot.title, "bookmarked_at": snapshot.bookmarked_at, "tags": snapshot.tags_str}
+            for snapshot in crawl.snapshot_set.all()
+        ]
+        xml = '<rss version="2.0"><channel>'
+        for url in urls:
+            xml += f"<item><url>{url['url']}</url><title>{url['title']}</title><bookmarked_at>{url['bookmarked_at']}</bookmarked_at><tags>{url['tags']}</tags></item>"
+        xml += "</channel></rss>"
+        return xml
+
+    return crawl
+
+
+@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
+def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
+    """Update a crawl (e.g., set status=sealed to cancel queued work)."""
+    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    payload = data.dict(exclude_unset=True)
+    update_fields = ["modified_at"]
+
+    tags = payload.pop("tags", None)
+    tags_str = payload.pop("tags_str", None)
+    if tags is not None or tags_str is not None:
+        crawl.tags_str = ",".join(normalize_tag_list(tags, tags_str or ""))
+        update_fields.append("tags_str")
+
+    if "status" in payload:
+        if payload["status"] not in Crawl.StatusChoices.values:
+            raise HttpError(400, f"Invalid status: {payload['status']}")
+        crawl.status = payload["status"]
+        if crawl.status == Crawl.StatusChoices.SEALED and "retry_at" not in payload:
+            crawl.retry_at = None
+        update_fields.append("status")
+
+    if "retry_at" in payload:
+        crawl.retry_at = payload["retry_at"]
+        update_fields.append("retry_at")
+
+    crawl.save(update_fields=update_fields)
+
+    if payload.get("status") == Crawl.StatusChoices.SEALED:
+        Snapshot.objects.filter(
+            crawl=crawl,
+            status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
+        ).update(
+            status=Snapshot.StatusChoices.SEALED,
+            retry_at=None,
+            modified_at=timezone.now(),
+        )
+    return crawl
+
+
+@router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
+def delete_crawl(request: HttpRequest, crawl_id: str):
+    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl_id_str = str(crawl.id)
+    snapshot_count = crawl.snapshot_set.count()
+    deleted_count, _ = crawl.delete()
+    return {
+        "success": True,
+        "crawl_id": crawl_id_str,
+        "deleted_count": deleted_count,
+        "deleted_snapshots": snapshot_count,
+    }
@@ -0,0 +1,161 @@
+__package__ = "archivebox.api"
+
+from uuid import UUID
+from typing import Annotated
+from datetime import datetime
+
+from django.http import HttpRequest
+
+from ninja import FilterLookup, FilterSchema, Query, Router, Schema
+from ninja.pagination import paginate
+
+from archivebox.api.v1_core import CustomPagination
+
+
+router = Router(tags=["Machine and Dependencies"])
+
+
+# ============================================================================
+# Machine Schemas
+# ============================================================================
+
+
+class MachineSchema(Schema):
+    """Schema for Machine model."""
+
+    TYPE: str = "machine.Machine"
+    id: UUID
+    created_at: datetime
+    modified_at: datetime
+    guid: str
+    hostname: str
+    hw_in_docker: bool
+    hw_in_vm: bool
+    hw_manufacturer: str
+    hw_product: str
+    hw_uuid: str
+    os_arch: str
+    os_family: str
+    os_platform: str
+    os_release: str
+    os_kernel: str
+    stats: dict
+    num_uses_succeeded: int
+    num_uses_failed: int
+
+
+class MachineFilterSchema(FilterSchema):
+    id: Annotated[str | None, FilterLookup("id__startswith")] = None
+    hostname: Annotated[str | None, FilterLookup("hostname__icontains")] = None
+    os_platform: Annotated[str | None, FilterLookup("os_platform__icontains")] = None
+    os_arch: Annotated[str | None, FilterLookup("os_arch")] = None
+    hw_in_docker: Annotated[bool | None, FilterLookup("hw_in_docker")] = None
+    hw_in_vm: Annotated[bool | None, FilterLookup("hw_in_vm")] = None
+    bin_providers: Annotated[str | None, FilterLookup("bin_providers__icontains")] = None
+
+
+# ============================================================================
+# Binary Schemas
+# ============================================================================
+
+
+class BinarySchema(Schema):
+    """Schema for Binary model."""
+
+    TYPE: str = "machine.Binary"
+    id: UUID
+    created_at: datetime
+    modified_at: datetime
+    machine_id: UUID
+    machine_hostname: str
+    name: str
+    binproviders: str
+    binprovider: str
+    abspath: str
+    version: str
+    sha256: str
+    status: str
+    is_valid: bool
+    num_uses_succeeded: int
+    num_uses_failed: int
+
+    @staticmethod
+    def resolve_machine_hostname(obj) -> str:
+        return obj.machine.hostname
+
+    @staticmethod
+    def resolve_is_valid(obj) -> bool:
+        return obj.is_valid
+
+
+class BinaryFilterSchema(FilterSchema):
+    id: Annotated[str | None, FilterLookup("id__startswith")] = None
+    name: Annotated[str | None, FilterLookup("name__icontains")] = None
+    binprovider: Annotated[str | None, FilterLookup("binprovider")] = None
+    status: Annotated[str | None, FilterLookup("status")] = None
+    machine_id: Annotated[str | None, FilterLookup("machine_id__startswith")] = None
+    version: Annotated[str | None, FilterLookup("version__icontains")] = None
+
+
+# ============================================================================
+# Machine Endpoints
+# ============================================================================
+
+
+@router.get("/machines", response=list[MachineSchema], url_name="get_machines")
+@paginate(CustomPagination)
+def get_machines(request: HttpRequest, filters: Query[MachineFilterSchema]):
+    """List all machines."""
+    from archivebox.machine.models import Machine
+
+    return filters.filter(Machine.objects.all()).distinct()
+
+
+@router.get("/machine/current", response=MachineSchema, url_name="get_current_machine")
+def get_current_machine(request: HttpRequest):
+    """Get the current machine."""
+    from archivebox.machine.models import Machine
+
+    return Machine.current()
+
+
+@router.get("/machine/{machine_id}", response=MachineSchema, url_name="get_machine")
+def get_machine(request: HttpRequest, machine_id: str):
+    """Get a specific machine by ID."""
+    from archivebox.machine.models import Machine
+    from django.db.models import Q
+
+    return Machine.objects.get(Q(id__startswith=machine_id) | Q(hostname__iexact=machine_id))
+
+
+# ============================================================================
+
+
+# ============================================================================
+# Binary Endpoints
+# ============================================================================
+
+
+@router.get("/binaries", response=list[BinarySchema], url_name="get_binaries")
+@paginate(CustomPagination)
+def get_binaries(request: HttpRequest, filters: Query[BinaryFilterSchema]):
+    """List all binaries."""
+    from archivebox.machine.models import Binary
+
+    return filters.filter(Binary.objects.all().select_related("machine")).distinct()
+
+
+@router.get("/binary/{binary_id}", response=BinarySchema, url_name="get_binary")
+def get_binary(request: HttpRequest, binary_id: str):
+    """Get a specific binary by ID."""
+    from archivebox.machine.models import Binary
+
+    return Binary.objects.select_related("machine").get(id__startswith=binary_id)
+
+
+@router.get("/binary/by-name/{name}", response=list[BinarySchema], url_name="get_binaries_by_name")
+def get_binaries_by_name(request: HttpRequest, name: str):
+    """Get all binaries with the given name."""
+    from archivebox.machine.models import Binary
+
+    return list(Binary.objects.filter(name__iexact=name).select_related("machine"))
@@ -0,0 +1 @@
+__package__ = "archivebox.base_models"
@@ -0,0 +1,726 @@
+"""Base admin classes for models using UUIDv7."""
+
+__package__ = "archivebox.base_models"
+
+import json
+from collections.abc import Mapping
+from typing import NotRequired, TypedDict
+
+from django import forms
+from django.contrib import admin
+from django.db import models
+from django.forms.renderers import BaseRenderer
+from django.http import HttpRequest, QueryDict
+from django.utils.safestring import SafeString, mark_safe
+from django_object_actions import DjangoObjectActions
+
+
+class ConfigOption(TypedDict):
+    plugin: str
+    type: str | list[str]
+    default: object
+    description: str
+    enum: NotRequired[list[object]]
+    pattern: NotRequired[str]
+    minimum: NotRequired[int | float]
+    maximum: NotRequired[int | float]
+
+
+class KeyValueWidget(forms.Widget):
+    """
+    A widget that renders JSON dict as editable key-value input fields
+    with + and - buttons to add/remove rows.
+    Includes autocomplete for available config keys from the plugin system.
+    """
+
+    template_name = ""  # We render manually
+
+    class Media:
+        css = {
+            "all": [],
+        }
+        js = []
+
+    def _get_config_options(self) -> dict[str, ConfigOption]:
+        """Get available config options from plugins."""
+        try:
+            from archivebox.hooks import discover_plugin_configs
+
+            plugin_configs = discover_plugin_configs()
+            options: dict[str, ConfigOption] = {}
+            for plugin_name, schema in plugin_configs.items():
+                for key, prop in schema.get("properties", {}).items():
+                    option: ConfigOption = {
+                        "plugin": plugin_name,
+                        "type": prop.get("type", "string"),
+                        "default": prop.get("default", ""),
+                        "description": prop.get("description", ""),
+                    }
+                    for schema_key in ("enum", "pattern", "minimum", "maximum"):
+                        if schema_key in prop:
+                            option[schema_key] = prop[schema_key]
+                    options[key] = option
+            return options
+        except Exception:
+            return {}
+
+    def _parse_value(self, value: object) -> dict[str, object]:
+        # Parse JSON value to dict
+        if value is None:
+            return {}
+        if isinstance(value, str):
+            try:
+                parsed = json.loads(value) if value else {}
+            except json.JSONDecodeError:
+                return {}
+            return parsed if isinstance(parsed, dict) else {}
+        if isinstance(value, Mapping):
+            return {str(key): item for key, item in value.items()}
+        return {}
+
+    def render(
+        self,
+        name: str,
+        value: object,
+        attrs: Mapping[str, str] | None = None,
+        renderer: BaseRenderer | None = None,
+    ) -> SafeString:
+        data = self._parse_value(value)
+
+        widget_id = attrs.get("id", name) if attrs else name
+        config_options = self._get_config_options()
+
+        # Build datalist options
+        datalist_options = "\n".join(
+            f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
+            for key, opt in sorted(config_options.items())
+        )
+
+        # Build config metadata as JSON for JS
+        config_meta_json = json.dumps(config_options)
+
+        html = f'''
+        <div id="{widget_id}_container" class="key-value-editor" style="max-width: 700px;">
+            <datalist id="{widget_id}_keys">
+                {datalist_options}
+            </datalist>
+            <div id="{widget_id}_rows" class="key-value-rows">
+        '''
+
+        # Render existing key-value pairs
+        for key, val in data.items():
+            val_str = json.dumps(val) if not isinstance(val, str) else val
+            html += self._render_row(widget_id, key, val_str)
+
+        # Always add one empty row for new entries
+        html += self._render_row(widget_id, "", "")
+
+        html += f'''
+            </div>
+            <div style="display: flex; gap: 8px; align-items: center; margin-top: 8px;">
+                <button type="button" onclick="addKeyValueRow_{widget_id}()"
+                        style="padding: 4px 12px; cursor: pointer; background: #417690; color: white; border: none; border-radius: 4px;">
+                    + Add Row
+                </button>
+            </div>
+            <input type="hidden" name="{name}" id="{widget_id}" value="">
+            <script>
+                (function() {{
+                    var configMeta_{widget_id} = {config_meta_json};
+                    var rowCounter_{widget_id} = 0;
+
+                    function stringifyValue_{widget_id}(value) {{
+                        return typeof value === 'string' ? value : JSON.stringify(value);
+                    }}
+
+                    function getTypes_{widget_id}(meta) {{
+                        if (!meta || meta.type === undefined || meta.type === null) {{
+                            return [];
+                        }}
+                        return Array.isArray(meta.type) ? meta.type : [meta.type];
+                    }}
+
+                    function getMetaForKey_{widget_id}(key) {{
+                        if (!key) {{
+                            return null;
+                        }}
+
+                        var explicitMeta = configMeta_{widget_id}[key];
+                        if (explicitMeta) {{
+                            return Object.assign({{ key: key }}, explicitMeta);
+                        }}
+
+                        if (key.endsWith('_BINARY')) {{
+                            return {{
+                                key: key,
+                                plugin: 'custom',
+                                type: 'string',
+                                default: '',
+                                description: 'Path to binary executable',
+                            }};
+                        }}
+
+                        if (isRegexConfigKey_{widget_id}(key)) {{
+                            return {{
+                                key: key,
+                                plugin: 'custom',
+                                type: 'string',
+                                default: '',
+                                description: 'Regex pattern list',
+                            }};
+                        }}
+
+                        return null;
+                    }}
+
+                    function describeMeta_{widget_id}(meta) {{
+                        if (!meta) {{
+                            return '';
+                        }}
+
+                        var details = '';
+                        if (Array.isArray(meta.enum) && meta.enum.length) {{
+                            details = 'Allowed: ' + meta.enum.map(stringifyValue_{widget_id}).join(', ');
+                        }} else {{
+                            var types = getTypes_{widget_id}(meta);
+                            if (types.length) {{
+                                details = 'Expected: ' + types.join(' or ');
+                            }}
+                        }}
+
+                        if (meta.minimum !== undefined || meta.maximum !== undefined) {{
+                            var bounds = [];
+                            if (meta.minimum !== undefined) bounds.push('min ' + meta.minimum);
+                            if (meta.maximum !== undefined) bounds.push('max ' + meta.maximum);
+                            details += (details ? ' ' : '') + '(' + bounds.join(', ') + ')';
+                        }}
+
+                        return [meta.description || '', details].filter(Boolean).join(' ');
+                    }}
+
+                    function getExampleInput_{widget_id}(key, meta) {{
+                        var types = getTypes_{widget_id}(meta);
+                        if (key.endsWith('_BINARY')) {{
+                            return 'Example: wget or /usr/bin/wget';
+                        }}
+                        if (key.endsWith('_ARGS_EXTRA') || key.endsWith('_ARGS')) {{
+                            return 'Example: ["--extra-arg"]';
+                        }}
+                        if (types.includes('array')) {{
+                            return 'Example: ["value"]';
+                        }}
+                        if (types.includes('object')) {{
+                            if (key === 'SAVE_ALLOWLIST' || key === 'SAVE_DENYLIST') {{
+                                return 'Example: {{"^https://example\\\\.com": ["wget"]}}';
+                            }}
+                            return 'Example: {{"key": "value"}}';
+                        }}
+                        return '';
+                    }}
+
+                    function isRegexConfigKey_{widget_id}(key) {{
+                        return key === 'URL_ALLOWLIST' ||
+                            key === 'URL_DENYLIST' ||
+                            key === 'SAVE_ALLOWLIST' ||
+                            key === 'SAVE_DENYLIST' ||
+                            key.endsWith('_PATTERN') ||
+                            key.includes('REGEX');
+                    }}
+
+                    function isSimpleFilterPattern_{widget_id}(pattern) {{
+                        return /^[\\w.*:-]+$/.test(pattern);
+                    }}
+
+                    function validateRegexPattern_{widget_id}(pattern) {{
+                        if (!pattern || isSimpleFilterPattern_{widget_id}(pattern)) {{
+                            return '';
+                        }}
+
+                        try {{
+                            new RegExp(pattern);
+                        }} catch (error) {{
+                            return error && error.message ? error.message : 'Invalid regex';
+                        }}
+                        return '';
+                    }}
+
+                    function validateRegexConfig_{widget_id}(key, raw, typeName) {{
+                        if (typeName === 'object') {{
+                            var parsed;
+                            try {{
+                                parsed = JSON.parse(raw);
+                            }} catch (error) {{
+                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
+                            }}
+                            if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {{
+                                return {{ ok: false, value: parsed, message: 'Must be a JSON object' }};
+                            }}
+                            for (var regexKey in parsed) {{
+                                var objectRegexError = validateRegexPattern_{widget_id}(regexKey);
+                                if (objectRegexError) {{
+                                    return {{ ok: false, value: parsed, message: 'Invalid regex key "' + regexKey + '": ' + objectRegexError }};
+                                }}
+                            }}
+                            return {{ ok: true, value: parsed, message: '' }};
+                        }}
+
+                        var patterns = raw.split(/[\\n,]+/).map(function(pattern) {{
+                            return pattern.trim();
+                        }}).filter(Boolean);
+                        for (var i = 0; i < patterns.length; i++) {{
+                            var regexError = validateRegexPattern_{widget_id}(patterns[i]);
+                            if (regexError) {{
+                                return {{ ok: false, value: raw, message: 'Invalid regex "' + patterns[i] + '": ' + regexError }};
+                            }}
+                        }}
+                        return {{ ok: true, value: raw, message: '' }};
+                    }}
+
+                    function validateBinaryValue_{widget_id}(raw) {{
+                        if (!raw) {{
+                            return {{ ok: true, value: raw, message: '' }};
+                        }}
+
+                        if (/['"`]/.test(raw)) {{
+                            return {{ ok: false, value: raw, message: 'Binary paths cannot contain quotes' }};
+                        }}
+
+                        if (/[;&|<>$(){{}}\\[\\]!]/.test(raw)) {{
+                            return {{ ok: false, value: raw, message: 'Binary paths can only be a binary name or absolute path' }};
+                        }}
+
+                        if (raw.startsWith('/')) {{
+                            if (/^[A-Za-z0-9_./+\\- ]+$/.test(raw)) {{
+                                return {{ ok: true, value: raw, message: '' }};
+                            }}
+                            return {{ ok: false, value: raw, message: 'Absolute paths may only contain path-safe characters' }};
+                        }}
+
+                        if (/^[A-Za-z0-9_.+-]+$/.test(raw)) {{
+                            return {{ ok: true, value: raw, message: '' }};
+                        }}
+
+                        return {{ ok: false, value: raw, message: 'Enter a binary name like wget or an absolute path like /usr/bin/wget' }};
+                    }}
+
+                    function parseValue_{widget_id}(raw) {{
+                        try {{
+                            if (raw === 'true') return true;
+                            if (raw === 'false') return false;
+                            if (raw === 'null') return null;
+                            if (raw !== '' && !isNaN(raw)) return Number(raw);
+                            if ((raw.startsWith('{{') && raw.endsWith('}}')) ||
+                                (raw.startsWith('[') && raw.endsWith(']')) ||
+                                (raw.startsWith('"') && raw.endsWith('"'))) {{
+                                return JSON.parse(raw);
+                            }}
+                        }} catch (error) {{
+                            return raw;
+                        }}
+                        return raw;
+                    }}
+
+                    function sameValue_{widget_id}(left, right) {{
+                        return left === right || JSON.stringify(left) === JSON.stringify(right);
+                    }}
+
+                    function parseTypedValue_{widget_id}(raw, typeName, meta) {{
+                        var numberValue;
+                        var parsed;
+
+                        if (typeName && meta && meta.key && isRegexConfigKey_{widget_id}(meta.key)) {{
+                            return validateRegexConfig_{widget_id}(meta.key, raw, typeName);
+                        }}
+
+                        if (typeName === 'string' && meta && meta.key && meta.key.endsWith('_BINARY')) {{
+                            return validateBinaryValue_{widget_id}(raw);
+                        }}
+
+                        if (typeName === 'string') {{
+                            if (meta.pattern) {{
+                                try {{
+                                    if (!(new RegExp(meta.pattern)).test(raw)) {{
+                                        return {{ ok: false, value: raw, message: 'Must match pattern ' + meta.pattern }};
+                                    }}
+                                }} catch (error) {{}}
+                            }}
+                            return {{ ok: true, value: raw, message: '' }};
+                        }}
+
+                        if (typeName === 'integer') {{
+                            if (!/^-?\\d+$/.test(raw)) {{
+                                return {{ ok: false, value: raw, message: 'Must be an integer' }};
+                            }}
+                            numberValue = Number(raw);
+                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
+                            }}
+                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
+                            }}
+                            return {{ ok: true, value: numberValue, message: '' }};
+                        }}
+
+                        if (typeName === 'number') {{
+                            if (raw === '' || isNaN(raw)) {{
+                                return {{ ok: false, value: raw, message: 'Must be a number' }};
+                            }}
+                            numberValue = Number(raw);
+                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
+                            }}
+                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
+                            }}
+                            return {{ ok: true, value: numberValue, message: '' }};
+                        }}
+
+                        if (typeName === 'boolean') {{
+                            var lowered = raw.toLowerCase();
+                            if (lowered === 'true' || raw === '1') return {{ ok: true, value: true, message: '' }};
+                            if (lowered === 'false' || raw === '0') return {{ ok: true, value: false, message: '' }};
+                            return {{ ok: false, value: raw, message: 'Must be true or false' }};
+                        }}
+
+                        if (typeName === 'null') {{
+                            return raw === 'null'
+                                ? {{ ok: true, value: null, message: '' }}
+                                : {{ ok: false, value: raw, message: 'Must be null' }};
+                        }}
+
+                        if (typeName === 'array' || typeName === 'object') {{
+                            try {{
+                                parsed = JSON.parse(raw);
+                            }} catch (error) {{
+                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
+                            }}
+
+                            if (typeName === 'array' && Array.isArray(parsed)) {{
+                                return {{ ok: true, value: parsed, message: '' }};
+                            }}
+                            if (typeName === 'object' && parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {{
+                                return {{ ok: true, value: parsed, message: '' }};
+                            }}
+
+                            return {{
+                                ok: false,
+                                value: parsed,
+                                message: typeName === 'array' ? 'Must be a JSON array' : 'Must be a JSON object',
+                            }};
+                        }}
+
+                        return {{ ok: true, value: parseValue_{widget_id}(raw), message: '' }};
+                    }}
+
+                    function validateValueAgainstMeta_{widget_id}(raw, meta) {{
+                        if (!meta || raw === '') {{
+                            return {{ state: 'neutral', value: raw, message: '' }};
+                        }}
+
+                        var enumValues = Array.isArray(meta.enum) ? meta.enum : [];
+                        var types = getTypes_{widget_id}(meta);
+                        if (!types.length) {{
+                            types = ['string'];
+                        }}
+
+                        var error = 'Invalid value';
+                        for (var i = 0; i < types.length; i++) {{
+                            var candidate = parseTypedValue_{widget_id}(raw, types[i], meta);
+                            if (!candidate.ok) {{
+                                error = candidate.message || error;
+                                continue;
+                            }}
+                            if (enumValues.length && !enumValues.some(function(enumValue) {{
+                                return sameValue_{widget_id}(enumValue, candidate.value) || stringifyValue_{widget_id}(enumValue) === raw;
+                            }})) {{
+                                error = 'Must be one of: ' + enumValues.map(stringifyValue_{widget_id}).join(', ');
+                                continue;
+                            }}
+                            return {{ state: 'valid', value: candidate.value, message: '' }};
+                        }}
+
+                        return {{ state: 'invalid', value: raw, message: error }};
+                    }}
+
+                    function ensureRowId_{widget_id}(row) {{
+                        if (!row.dataset.rowId) {{
+                            row.dataset.rowId = String(rowCounter_{widget_id}++);
+                        }}
+                        return row.dataset.rowId;
+                    }}
+
+                    function setRowHelp_{widget_id}(row) {{
+                        var keyInput = row.querySelector('.kv-key');
+                        var help = row.querySelector('.kv-help');
+                        if (!keyInput || !help) {{
+                            return;
+                        }}
+
+                        var key = keyInput.value.trim();
+                        if (!key) {{
+                            help.textContent = '';
+                            return;
+                        }}
+
+                        var meta = getMetaForKey_{widget_id}(key);
+                        if (meta) {{
+                            var extra = isRegexConfigKey_{widget_id}(key)
+                                ? ((meta.type === 'object' || (Array.isArray(meta.type) && meta.type.includes('object')))
+                                    ? ' Expected: JSON object with regex keys.'
+                                    : ' Expected: valid regex.')
+                                : '';
+                            var example = getExampleInput_{widget_id}(key, meta);
+                            help.textContent = [describeMeta_{widget_id}(meta) + extra, example].filter(Boolean).join(' ');
+                        }} else {{
+                            help.textContent = 'Custom key';
+                        }}
+                    }}
+
+                    function configureValueInput_{widget_id}(row) {{
+                        var keyInput = row.querySelector('.kv-key');
+                        var valueInput = row.querySelector('.kv-value');
+                        var datalist = row.querySelector('.kv-value-options');
+                        if (!keyInput || !valueInput || !datalist) {{
+                            return;
+                        }}
+
+                        var rowId = ensureRowId_{widget_id}(row);
+                        datalist.id = '{widget_id}_value_options_' + rowId;
+
+                        var meta = getMetaForKey_{widget_id}(keyInput.value.trim());
+                        var enumValues = Array.isArray(meta && meta.enum) ? meta.enum : [];
+                        var types = getTypes_{widget_id}(meta);
+                        if (!enumValues.length && types.includes('boolean')) {{
+                            enumValues = ['True', 'False'];
+                        }}
+                        if (enumValues.length) {{
+                            datalist.innerHTML = enumValues.map(function(enumValue) {{
+                                return '<option value="' + stringifyValue_{widget_id}(enumValue).replace(/"/g, '&quot;') + '"></option>';
+                            }}).join('');
+                            valueInput.setAttribute('list', datalist.id);
+                        }} else {{
+                            datalist.innerHTML = '';
+                            valueInput.removeAttribute('list');
+                        }}
+                    }}
+
+                    function setValueValidationState_{widget_id}(input, state, message) {{
+                        if (!input) {{
+                            return;
+                        }}
+
+                        if (state === 'valid') {{
+                            input.style.borderColor = '#2da44e';
+                            input.style.boxShadow = '0 0 0 1px rgba(45, 164, 78, 0.18)';
+                            input.style.backgroundColor = '#f6ffed';
+                        }} else if (state === 'invalid') {{
+                            input.style.borderColor = '#cf222e';
+                            input.style.boxShadow = '0 0 0 1px rgba(207, 34, 46, 0.18)';
+                            input.style.backgroundColor = '#fff8f8';
+                        }} else {{
+                            input.style.borderColor = '#ccc';
+                            input.style.boxShadow = 'none';
+                            input.style.backgroundColor = '';
+                        }}
+                        input.title = message || '';
+                    }}
+
+                    function applyValueValidation_{widget_id}(row) {{
+                        var keyInput = row.querySelector('.kv-key');
+                        var valueInput = row.querySelector('.kv-value');
+                        if (!keyInput || !valueInput) {{
+                            return;
+                        }}
+
+                        var key = keyInput.value.trim();
+                        if (!key) {{
+                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
+                            return;
+                        }}
+
+                        var meta = getMetaForKey_{widget_id}(key);
+                        if (!meta) {{
+                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
+                            return;
+                        }}
+
+                        var validation = validateValueAgainstMeta_{widget_id}(valueInput.value.trim(), meta);
+                        setValueValidationState_{widget_id}(valueInput, validation.state, validation.message);
+                    }}
+
+                    function coerceValueForStorage_{widget_id}(key, raw) {{
+                        var meta = getMetaForKey_{widget_id}(key);
+                        if (!meta) {{
+                            return parseValue_{widget_id}(raw);
+                        }}
+
+                        var validation = validateValueAgainstMeta_{widget_id}(raw, meta);
+                        return validation.state === 'valid' ? validation.value : raw;
+                    }}
+
+                    function initializeRows_{widget_id}() {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
+                            ensureRowId_{widget_id}(row);
+                            configureValueInput_{widget_id}(row);
+                            setRowHelp_{widget_id}(row);
+                            applyValueValidation_{widget_id}(row);
+                        }});
+                    }}
+
+                    function updateHiddenField_{widget_id}() {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        var rows = container.querySelectorAll('.key-value-row');
+                        var result = {{}};
+                        rows.forEach(function(row) {{
+                            var keyInput = row.querySelector('.kv-key');
+                            var valInput = row.querySelector('.kv-value');
+                            if (keyInput && valInput && keyInput.value.trim()) {{
+                                var key = keyInput.value.trim();
+                                var val = valInput.value.trim();
+                                result[key] = coerceValueForStorage_{widget_id}(key, val);
+                            }}
+                        }});
+                        document.getElementById('{widget_id}').value = JSON.stringify(result);
+                    }}
+
+                    window.addKeyValueRow_{widget_id} = function() {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        var newRow = document.createElement('div');
+                        newRow.className = 'key-value-row';
+                        newRow.style.cssText = 'margin-bottom: 6px;';
+                        newRow.innerHTML = '<div style="display: flex; gap: 8px; align-items: center;">' +
+                            '<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
+                            'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
+                            '<input type="text" class="kv-value" placeholder="value" ' +
+                            'style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
+                            '<datalist class="kv-value-options"></datalist>' +
+                            '<button type="button" onclick="removeKeyValueRow_{widget_id}(this)" ' +
+                            'style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>' +
+                            '</div>' +
+                            '<div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>';
+                        container.appendChild(newRow);
+                        ensureRowId_{widget_id}(newRow);
+                        configureValueInput_{widget_id}(newRow);
+                        setRowHelp_{widget_id}(newRow);
+                        applyValueValidation_{widget_id}(newRow);
+                        updateHiddenField_{widget_id}();
+                        newRow.querySelector('.kv-key').focus();
+                    }};
+
+                    window.removeKeyValueRow_{widget_id} = function(btn) {{
+                        var row = btn.closest('.key-value-row');
+                        row.remove();
+                        updateHiddenField_{widget_id}();
+                    }};
+
+                    window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
+
+                    // Initialize on load
+                    document.addEventListener('DOMContentLoaded', function() {{
+                        initializeRows_{widget_id}();
+                        updateHiddenField_{widget_id}();
+                    }});
+                    // Also run immediately in case DOM is already ready
+                    if (document.readyState !== 'loading') {{
+                        initializeRows_{widget_id}();
+                        updateHiddenField_{widget_id}();
+                    }}
+
+                    // Update on any input change
+                    var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
+
+                    rowsEl_{widget_id}.addEventListener('input', function(event) {{
+                        var row = event.target.closest('.key-value-row');
+                        if (!row) {{
+                            return;
+                        }}
+
+                        if (event.target.classList.contains('kv-key')) {{
+                            configureValueInput_{widget_id}(row);
+                            setRowHelp_{widget_id}(row);
+                        }}
+
+                        if (event.target.classList.contains('kv-key') || event.target.classList.contains('kv-value')) {{
+                            applyValueValidation_{widget_id}(row);
+                            updateHiddenField_{widget_id}();
+                        }}
+                    }});
+                }})();
+            </script>
+        </div>
+        '''
+        return mark_safe(html)
+
+    def _render_row(self, widget_id: str, key: str, value: str) -> str:
+        return f'''
+            <div class="key-value-row" style="margin-bottom: 6px;">
+                <div style="display: flex; gap: 8px; align-items: center;">
+                    <input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
+                           style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
+                    <input type="text" class="kv-value" value="{self._escape(value)}" placeholder="value"
+                           style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
+                    <datalist class="kv-value-options"></datalist>
+                    <button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
+                            style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>
+                </div>
+                <div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>
+            </div>
+        '''
+
+    def _escape(self, s: object) -> str:
+        """Escape HTML special chars in attribute values."""
+        if not s:
+            return ""
+        return str(s).replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
+
+    def value_from_datadict(
+        self,
+        data: QueryDict | Mapping[str, object],
+        files: object,
+        name: str,
+    ) -> str:
+        value = data.get(name, "{}")
+        return value if isinstance(value, str) else "{}"
+
+
+class ConfigEditorMixin(admin.ModelAdmin):
+    """
+    Mixin for admin classes with a config JSON field.
+
+    Provides a key-value editor widget with autocomplete for available config keys.
+    """
+
+    def formfield_for_dbfield(
+        self,
+        db_field: models.Field[object, object],
+        request: HttpRequest,
+        **kwargs: object,
+    ) -> forms.Field | None:
+        """Use KeyValueWidget for the config JSON field."""
+        if db_field.name == "config":
+            kwargs["widget"] = KeyValueWidget()
+        return super().formfield_for_dbfield(db_field, request, **kwargs)
+
+
+class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
+    list_display = ("id", "created_at", "created_by")
+    readonly_fields = ("id", "created_at", "modified_at")
+    show_search_mode_selector = False
+
+    def get_default_search_mode(self) -> str:
+        # The shared changelist template always asks every admin for a default
+        # search mode, even when the search-mode toggle is hidden.
+        return "meta"
+
+    def get_form(
+        self,
+        request: HttpRequest,
+        obj: models.Model | None = None,
+        change: bool = False,
+        **kwargs: object,
+    ):
+        form = super().get_form(request, obj, change=change, **kwargs)
+        if "created_by" in form.base_fields:
+            form.base_fields["created_by"].initial = request.user
+        return form
@@ -0,0 +1,7 @@
+# from django.apps import AppConfig
+
+
+# class BaseModelsConfig(AppConfig):
+#     default_auto_field = 'django.db.models.BigAutoField'
+
+#     name = 'base_models'
@@ -0,0 +1,134 @@
+"""Base models using UUIDv7 for all id fields."""
+
+__package__ = "archivebox.base_models"
+
+from archivebox.uuid_compat import uuid7
+from pathlib import Path
+
+from django.db import models
+from django.db.models import F
+from django.utils import timezone
+from django.contrib.auth import get_user_model
+from django.urls import reverse_lazy
+from django.conf import settings
+
+from django_stubs_ext.db.models import TypedModelMeta
+
+
+def get_or_create_system_user_pk(username="system"):
+    User = get_user_model()
+    # If there's exactly one superuser, use that for all system operations
+    if User.objects.filter(is_superuser=True).count() == 1:
+        return User.objects.filter(is_superuser=True).values_list("pk", flat=True)[0]
+    # Otherwise get or create the system user
+    user, _ = User.objects.get_or_create(
+        username=username,
+        defaults={"is_staff": True, "is_superuser": True, "email": "", "password": "!"},
+    )
+    return user.pk
+
+
+class AutoDateTimeField(models.DateTimeField):
+    """DateTimeField that automatically updates on save (legacy compatibility)."""
+
+    def pre_save(self, model_instance, add):
+        if add or not getattr(model_instance, self.attname):
+            value = timezone.now()
+            setattr(model_instance, self.attname, value)
+            return value
+        return super().pre_save(model_instance, add)
+
+
+class ModelWithUUID(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    created_at = models.DateTimeField(default=timezone.now, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
+    created_by = models.ForeignKey(
+        settings.AUTH_USER_MODEL,
+        on_delete=models.CASCADE,
+        default=get_or_create_system_user_pk,
+        null=False,
+        db_index=True,
+    )
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    def __str__(self) -> str:
+        return f"[{self.id}] {self.__class__.__name__}"
+
+    @property
+    def admin_change_url(self) -> str:
+        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
+
+    @property
+    def api_url(self) -> str:
+        return str(reverse_lazy("api-1:get_any", args=[self.id]))
+
+    @property
+    def api_docs_url(self) -> str:
+        return f"/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}"
+
+
+class ModelWithNotes(models.Model):
+    """Mixin for models with a notes field."""
+
+    notes = models.TextField(blank=True, null=False, default="")
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+
+class ModelWithHealthStats(models.Model):
+    """Mixin for models with health tracking fields."""
+
+    num_uses_failed = models.PositiveIntegerField(default=0)
+    num_uses_succeeded = models.PositiveIntegerField(default=0)
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    @property
+    def health(self) -> int:
+        total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
+        return round((self.num_uses_succeeded / total) * 100)
+
+    def increment_health_stats(self, success: bool):
+        """Atomically increment success or failure counter using F() expression."""
+        field = "num_uses_succeeded" if success else "num_uses_failed"
+        type(self).objects.filter(pk=self.pk).update(**{field: F(field) + 1})
+
+
+class ModelWithConfig(models.Model):
+    """Mixin for models with a JSON config field."""
+
+    config = models.JSONField(default=dict, null=True, blank=True, editable=True)
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+
+class ModelWithOutputDir(ModelWithUUID):
+    class Meta(ModelWithUUID.Meta):
+        abstract = True
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        Path(self.output_dir).mkdir(parents=True, exist_ok=True)
+        # Note: index.json is deprecated, models should use write_index_jsonl() for full data
+
+    @property
+    def output_dir_parent(self) -> str:
+        return f"{self._meta.model_name}s"
+
+    @property
+    def output_dir_name(self) -> str:
+        return str(self.id)
+
+    @property
+    def output_dir_str(self) -> str:
+        return f"{self.output_dir_parent}/{self.output_dir_name}"
+
+    @property
+    def output_dir(self) -> Path:
+        raise NotImplementedError(f"{self.__class__.__name__} must implement output_dir property")
@@ -1,264 +1,199 @@
-__package__ = 'archivebox.cli'
-__command__ = 'archivebox'
-
+__package__ = "archivebox.cli"
+__command__ = "archivebox"
 import os
 import sys
-import argparse
-import threading
-
-from time import sleep
-from collections.abc import Mapping
-
-from rich import print
-
-from typing import Optional, List, IO, Union, Iterable
-from pathlib import Path
-
 from importlib import import_module

-BUILTIN_LIST = list
+import rich_click as click
+from rich import print

-CLI_DIR = Path(__file__).resolve().parent
-
-# rewrite setup -> install for backwards compatibility
-if len(sys.argv) > 1 and sys.argv[1] == 'setup':
-    from rich import print
-    print(':warning: [bold red]DEPRECATED[/bold red] `archivebox setup` is deprecated, use `archivebox install` instead')
-    sys.argv[1] = 'install'
-
-if '--debug' in sys.argv:
-    os.environ['DEBUG'] = 'True'
-    sys.argv.remove('--debug')
+from archivebox.config.version import VERSION


-# def list_subcommands() -> Dict[str, str]:
-#     """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
-#     COMMANDS = []
-#     for filename in os.listdir(CLI_DIR):
-#         if is_cli_module(filename):
-#             subcommand = filename.replace('archivebox_', '').replace('.py', '')
-#             module = import_module('.archivebox_{}'.format(subcommand), __package__)
-#             assert is_valid_cli_module(module, subcommand)
-#             COMMANDS.append((subcommand, module.main.__doc__))
-#             globals()[subcommand] = module.main
-#     display_order = lambda cmd: (
-#         display_first.index(cmd[0])
-#         if cmd[0] in display_first else
-#         100 + len(cmd[0])
-#     )
-#     return dict(sorted(COMMANDS, key=display_order))
-
-# just define it statically, it's much faster:
-SUBCOMMAND_MODULES = {
-    'help': 'archivebox_help',
-    'version': 'archivebox_version' ,
-    
-    'init': 'archivebox_init',
-    'install': 'archivebox_install',
-    ##############################################
-    'config': 'archivebox_config',
-    'add': 'archivebox_add',
-    'remove': 'archivebox_remove',
-    'update': 'archivebox_update',
-    'list': 'archivebox_list',
-    'status': 'archivebox_status',
-    
-    'schedule': 'archivebox_schedule',
-    'server': 'archivebox_server',
-    'shell': 'archivebox_shell',
-    'manage': 'archivebox_manage',
-
-    # 'oneshot': 'archivebox_oneshot',
-}
-
-# every imported command module must have these properties in order to be valid
-required_attrs = ('__package__', '__command__', 'main')
-
-# basic checks to make sure imported files are valid subcommands
-is_cli_module = lambda fname: fname.startswith('archivebox_') and fname.endswith('.py')
-is_valid_cli_module = lambda module, subcommand: (
-    all(hasattr(module, attr) for attr in required_attrs)
-    and module.__command__.split(' ')[-1] == subcommand
-)
-
-class LazySubcommands(Mapping):
-    def keys(self):
-        return SUBCOMMAND_MODULES.keys()
-    
-    def values(self):
-        return [self[key] for key in self.keys()]
-    
-    def items(self):
-        return [(key, self[key]) for key in self.keys()]
-    
-    def __getitem__(self, key):
-        module = import_module(f'.{SUBCOMMAND_MODULES[key]}', __package__)
-        assert is_valid_cli_module(module, key)
-        return module.main
-    
-    def __iter__(self):
-        return iter(SUBCOMMAND_MODULES.keys())
-    
-    def __len__(self):
-        return len(SUBCOMMAND_MODULES)
-
-CLI_SUBCOMMANDS = LazySubcommands()
+if "--debug" in sys.argv:
+    os.environ["DEBUG"] = "True"
+    sys.argv.remove("--debug")


-# these common commands will appear sorted before any others for ease-of-use
-meta_cmds = ('help', 'version')                               # dont require valid data folder at all
-setup_cmds = ('init', 'setup', 'install')                      # require valid data folder, but dont require DB present in it yet
-archive_cmds = ('add', 'remove', 'update', 'list', 'status', 'schedule', 'server', 'shell', 'manage')  # require valid data folder + existing db present
-fake_db = ("oneshot",)                                        # use fake in-memory db
+class ArchiveBoxGroup(click.Group):
+    """lazy loading click group for archivebox commands"""

-display_first = (*meta_cmds, *setup_cmds, *archive_cmds)
+    meta_commands = {
+        "help": "archivebox.cli.archivebox_help.main",
+        "version": "archivebox.cli.archivebox_version.main",
+        "mcp": "archivebox.cli.archivebox_mcp.main",
+    }
+    setup_commands = {
+        "init": "archivebox.cli.archivebox_init.main",
+        "install": "archivebox.cli.archivebox_install.main",
+    }
+    # Model commands (CRUD operations via subcommands)
+    model_commands = {
+        "crawl": "archivebox.cli.archivebox_crawl.main",
+        "snapshot": "archivebox.cli.archivebox_snapshot.main",
+        "archiveresult": "archivebox.cli.archivebox_archiveresult.main",
+        "tag": "archivebox.cli.archivebox_tag.main",
+        "binary": "archivebox.cli.archivebox_binary.main",
+        "process": "archivebox.cli.archivebox_process.main",
+        "machine": "archivebox.cli.archivebox_machine.main",
+        "persona": "archivebox.cli.archivebox_persona.main",
+    }
+    archive_commands = {
+        # High-level commands
+        "add": "archivebox.cli.archivebox_add.main",
+        "extract": "archivebox.cli.archivebox_extract.main",
+        "list": "archivebox.cli.archivebox_list.main",
+        "remove": "archivebox.cli.archivebox_remove.main",
+        "run": "archivebox.cli.archivebox_run.main",
+        "update": "archivebox.cli.archivebox_update.main",
+        "status": "archivebox.cli.archivebox_status.main",
+        "search": "archivebox.cli.archivebox_search.main",
+        "config": "archivebox.cli.archivebox_config.main",
+        "schedule": "archivebox.cli.archivebox_schedule.main",
+        "server": "archivebox.cli.archivebox_server.main",
+        "shell": "archivebox.cli.archivebox_shell.main",
+        "manage": "archivebox.cli.archivebox_manage.main",
+        # Introspection commands
+        "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
+    }
+    legacy_model_commands = {
+        "crawl": "archivebox.cli.archivebox_crawl_compat.main",
+        "snapshot": "archivebox.cli.archivebox_snapshot_compat.main",
+    }
+    all_subcommands = {
+        **meta_commands,
+        **setup_commands,
+        **model_commands,
+        **archive_commands,
+    }
+    renamed_commands = {
+        "setup": "install",
+        "import": "add",
+        "archive": "add",
+    }
+    legacy_model_subcommands = {
+        "crawl": {"create", "list", "update", "delete"},
+        "snapshot": {"create", "list", "update", "delete"},
+    }
+
+    @classmethod
+    def get_canonical_name(cls, cmd_name):
+        return cls.renamed_commands.get(cmd_name, cmd_name)
+
+    @classmethod
+    def _should_use_legacy_model_command(cls, cmd_name: str) -> bool:
+        if cmd_name not in cls.legacy_model_commands:
+            return False
+
+        try:
+            arg_idx = sys.argv.index(cmd_name)
+        except ValueError:
+            return False
+
+        remaining_args = sys.argv[arg_idx + 1 :]
+        if not remaining_args:
+            return False
+
+        first_arg = remaining_args[0]
+        if first_arg in ("-h", "--help"):
+            return False
+
+        return first_arg not in cls.legacy_model_subcommands[cmd_name]
+
+    def get_command(self, ctx, cmd_name):
+        # handle renamed commands
+        if cmd_name in self.renamed_commands:
+            new_name = self.renamed_commands[cmd_name]
+            print(
+                f" [violet]Hint:[/violet] `archivebox {cmd_name}` has been renamed to `archivebox {new_name}`",
+                file=sys.stderr,
+            )
+            cmd_name = new_name
+            ctx.invoked_subcommand = cmd_name
+
+        if self._should_use_legacy_model_command(cmd_name):
+            return self._lazy_load(self.legacy_model_commands[cmd_name])
+
+        # handle lazy loading of commands
+        if cmd_name in self.all_subcommands:
+            return self._lazy_load(cmd_name)
+
+        # fall-back to using click's default command lookup
+        return super().get_command(ctx, cmd_name)
+
+    @classmethod
+    def _lazy_load(cls, cmd_name_or_path):
+        import_path = cls.all_subcommands.get(cmd_name_or_path)
+        if import_path is None:
+            import_path = cmd_name_or_path
+        modname, funcname = import_path.rsplit(".", 1)
+
+        # print(f'LAZY LOADING {import_path}')
+        mod = import_module(modname)
+        func = getattr(mod, funcname)
+
+        if not hasattr(func, "__doc__"):
+            raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
+
+        # if not isinstance(cmd, click.BaseCommand):
+        # raise ValueError(f'lazy loading of {import_path} failed - not a click command')
+
+        return func


-IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread', 'Scheduler')  # threads we dont have to wait for before exiting
+@click.group(cls=ArchiveBoxGroup, invoke_without_command=True)
+@click.option("--help", "-h", is_flag=True, help="Show help")
+@click.version_option(VERSION, "-v", "--version", package_name="archivebox", message="%(version)s")
+@click.pass_context
+def cli(ctx, help=False):
+    """ArchiveBox: The self-hosted internet archive"""
+
+    subcommand = ArchiveBoxGroup.get_canonical_name(ctx.invoked_subcommand)
+
+    # if --help is passed or no subcommand is given, show custom help message
+    if help or ctx.invoked_subcommand is None:
+        ctx.invoke(ctx.command.get_command(ctx, "help"))
+
+    # if the subcommand is in archive_commands or model_commands,
+    # then we need to set up the django environment and check that we're in a valid data folder
+    if subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands:
+        # print('SETUP DJANGO AND CHECK DATA FOLDER')
+        try:
+            if subcommand == "server":
+                run_in_debug = "--reload" in sys.argv or os.environ.get("DEBUG") in ("1", "true", "True", "TRUE", "yes")
+                if run_in_debug:
+                    os.environ["ARCHIVEBOX_RUNSERVER"] = "1"
+                    if "--reload" in sys.argv:
+                        os.environ["ARCHIVEBOX_AUTORELOAD"] = "1"
+                        from archivebox.config.common import STORAGE_CONFIG
+
+                        os.environ["ARCHIVEBOX_RUNSERVER_PIDFILE"] = str(STORAGE_CONFIG.TMP_DIR / "runserver.pid")
+
+            from archivebox.config.django import setup_django
+            from archivebox.misc.checks import check_data_folder
+
+            setup_django()
+            check_data_folder()
+        except Exception as e:
+            print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]", file=sys.stderr)
+            if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
+                raise


-def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: Iterable[str]=IGNORED_BG_THREADS, timeout: int=60) -> int:
-    """
-    Block until the specified threads exit. e.g. pass thread_names=('default_hook_handler',) to wait for webhooks.
-    Useful for waiting for signal handlers, webhooks, etc. to finish running after a mgmt command completes.
-    """
+def main(args=None, prog_name=None, stdin=None):
+    # show `docker run archivebox xyz` in help messages if running in docker
+    IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
+    IS_TTY = sys.stdin.isatty()
+    prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")

-    wait_for_all: bool = thread_names == ()
-
-    thread_matches = lambda thread, ptns: any(ptn in repr(thread) for ptn in ptns)
-
-    should_wait = lambda thread: (
-        not thread_matches(thread, ignore_names)
-        and (wait_for_all or thread_matches(thread, thread_names)))
-
-    for tries in range(timeout):
-        all_threads = [*threading.enumerate()]
-        blocking_threads = [*filter(should_wait, all_threads)]
-        threads_summary = ', '.join(repr(t) for t in blocking_threads)
-        if blocking_threads:
-            sleep(1)
-            if tries == 5:                            # only show stderr message if we need to wait more than 5s
-                print(
-                    f'[…] Waiting up to {timeout}s for background jobs (e.g. webhooks) to finish...',
-                    threads_summary,
-                    file=sys.stderr,
-                )
-        else:
-            return tries
-
-    raise Exception(f'Background threads failed to exit after {tries}s: {threads_summary}')
-
-
-
-def run_subcommand(subcommand: str,
-                   subcommand_args: List[str] | None = None,
-                   stdin: Optional[IO]=None,
-                   pwd: Union[Path, str, None]=None) -> None:
-    """Run a given ArchiveBox subcommand with the given list of args"""
-
-    subcommand_args = subcommand_args or []
-
-    from archivebox.misc.checks import check_migrations
-    from archivebox.config.django import setup_django
-    
-    # print('DATA_DIR is', DATA_DIR)
-    # print('pwd is', os.getcwd())    
-
-    cmd_requires_db = (subcommand in archive_cmds)
-    init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
-
-    check_db = cmd_requires_db and not init_pending
-
-    setup_django(in_memory_db=subcommand in fake_db, check_db=check_db)
-
-    for ignore_pattern in ('help', '-h', '--help', 'version', '--version'):
-        if ignore_pattern in sys.argv[:4]:
-            cmd_requires_db = False
-            break
-    
-    if subcommand in archive_cmds:
-        if cmd_requires_db:
-            check_migrations()
-
-    module = import_module('.archivebox_{}'.format(subcommand), __package__)
-    module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore
-
-    # wait for webhooks, signals, and other background jobs to finish before exit
-    wait_for_bg_threads_to_exit(timeout=60)
-
-
-
-
-
-class NotProvided:
-    def __len__(self):
-        return 0
-    def __bool__(self):
-        return False
-    def __repr__(self):
-        return '<not provided>'
-
-Omitted = Union[None, NotProvided]
-
-OMITTED = NotProvided()
-
-
-def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: str | None=None) -> None:
-    # print('STARTING CLI MAIN ENTRYPOINT')
-    
-    args = sys.argv[1:] if args is OMITTED else args
-    stdin = sys.stdin if stdin is OMITTED else stdin
-
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description='ArchiveBox: The self-hosted internet archive',
-        add_help=False,
-    )
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-        '--help', '-h',
-        action='store_true',
-        help=CLI_SUBCOMMANDS['help'].__doc__,
-    )
-    group.add_argument(
-        '--version',
-        action='store_true',
-        help=CLI_SUBCOMMANDS['version'].__doc__,
-    )
-    group.add_argument(
-        "subcommand",
-        type=str,
-        help= "The name of the subcommand to run",
-        nargs='?',
-        choices=CLI_SUBCOMMANDS.keys(),
-        default=None,
-    )
-    parser.add_argument(
-        "subcommand_args",
-        help="Arguments for the subcommand",
-        nargs=argparse.REMAINDER,
-    )
-    command = parser.parse_args(args or ())
-
-    if command.version:
-        command.subcommand = 'version'
-    elif command.help or command.subcommand is None:
-        command.subcommand = 'help'
-
-    if command.subcommand not in ('version',):
-        from ..logging_util import log_cli_command
-
-        log_cli_command(
-            subcommand=command.subcommand,
-            subcommand_args=command.subcommand_args,
-            stdin=stdin or None,
-        )
+    # stdin param allows passing input data from caller (used by __main__.py)
+    # currently not used by click-based CLI, but kept for backwards compatibility

    try:
-        run_subcommand(
-            subcommand=command.subcommand,
-            subcommand_args=command.subcommand_args,
-            stdin=stdin or None,
-        )
+        cli(args=args, prog_name=prog_name)
    except KeyboardInterrupt:
-        print('\n\n[red][X] Got CTRL+C. Exiting...[/red]')
+        print("\n\n[red][X] Got CTRL+C. Exiting...[/red]")
+
+
+if __name__ == "__main__":
+    main()
@@ -1,142 +1,286 @@
 #!/usr/bin/env python3

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox add'
+__package__ = "archivebox.cli"
+__command__ = "archivebox add"

 import sys
-import argparse
+from pathlib import Path

-from typing import List, Optional, IO
+from typing import TYPE_CHECKING

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from archivebox.config.common import ARCHIVING_CONFIG
+import rich_click as click

-from ..main import add
-from ..parsers import PARSERS
-from ..logging_util import SmartFormatter, accept_stdin, stderr
+from django.utils import timezone
+from django.db.models import QuerySet
+
+from archivebox.misc.util import enforce_types, docstring
+from archivebox.misc.util import parse_filesize_to_bytes
+from archivebox import CONSTANTS
+from archivebox.config.common import ARCHIVING_CONFIG, SERVER_CONFIG
+from archivebox.config.permissions import USER, HOSTNAME


+if TYPE_CHECKING:
+    from archivebox.core.models import Snapshot
+    from archivebox.crawls.models import Crawl
+
+
+def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
+    from archivebox.misc.jsonl import read_args_or_stdin
+
+    urls: list[str] = []
+    for record in read_args_or_stdin(args):
+        url = record.get("url")
+        if isinstance(url, str) and url:
+            urls.append(url)
+
+        urls_field = record.get("urls")
+        if isinstance(urls_field, str):
+            for line in urls_field.splitlines():
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    urls.append(line)
+
+    return urls
+
+
+@enforce_types
+def add(
+    urls: str | list[str],
+    depth: int | str = 0,
+    max_urls: int = 0,
+    max_size: int | str = 0,
+    tag: str = "",
+    url_allowlist: str = "",
+    url_denylist: str = "",
+    parser: str = "auto",
+    plugins: str = "",
+    persona: str = "Default",
+    overwrite: bool = False,
+    update: bool | None = None,
+    index_only: bool = False,
+    bg: bool = False,
+    created_by_id: int | None = None,
+) -> tuple["Crawl", QuerySet["Snapshot"]]:
+    """Add a new URL or list of URLs to your archive.
+
+    The flow is:
+    1. Save URLs to sources file
+    2. Create Crawl with URLs and max_depth
+    3. Crawl runner creates Snapshots from Crawl URLs (depth=0)
+    4. Crawl runner runs parser extractors on root snapshots
+    5. Parser extractors output to urls.jsonl
+    6. URLs are added to Crawl.urls and child Snapshots are created
+    7. Repeat until max_depth is reached
+    """
+
+    from rich import print
+
+    depth = int(depth)
+    max_urls = int(max_urls or 0)
+    max_size = parse_filesize_to_bytes(max_size)
+
+    if depth not in (0, 1, 2, 3, 4):
+        raise ValueError("Depth must be 0-4")
+    if max_urls < 0:
+        raise ValueError("max_urls must be >= 0")
+    if max_size < 0:
+        raise ValueError("max_size must be >= 0")
+
+    # import models once django is set up
+    from archivebox.core.models import Snapshot
+    from archivebox.crawls.models import Crawl
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.personas.models import Persona
+    from archivebox.misc.logging_util import printable_filesize
+    from archivebox.misc.system import get_dir_size
+    from archivebox.config.configset import get_config
+    from archivebox.services.runner import run_crawl
+
+    created_by_id = created_by_id or get_or_create_system_user_pk()
+    started_at = timezone.now()
+    if update is None:
+        update = not ARCHIVING_CONFIG.ONLY_NEW
+
+    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
+    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
+    sources_file.parent.mkdir(parents=True, exist_ok=True)
+    sources_file.write_text(urls if isinstance(urls, str) else "\n".join(urls))
+
+    # 2. Create a new Crawl with inline URLs
+    cli_args = [*sys.argv]
+    if cli_args[0].lower().endswith("archivebox"):
+        cli_args[0] = "archivebox"
+    cmd_str = " ".join(cli_args)
+
+    timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
+
+    # Read URLs directly into crawl
+    urls_content = sources_file.read_text()
+    persona_name = (persona or "Default").strip() or "Default"
+    plugins = plugins or str(get_config().get("PLUGINS") or "")
+    persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
+    persona_obj.ensure_dirs()
+
+    crawl = Crawl.objects.create(
+        urls=urls_content,
+        max_depth=depth,
+        max_urls=max_urls,
+        max_size=max_size,
+        tags_str=tag,
+        persona_id=persona_obj.id,
+        label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
+        created_by_id=created_by_id,
+        config={
+            "ONLY_NEW": not update,
+            "INDEX_ONLY": index_only,
+            "OVERWRITE": overwrite,
+            "PLUGINS": plugins,
+            "DEFAULT_PERSONA": persona_name,
+            "PARSER": parser,
+            **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
+            **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
+        },
+    )
+
+    print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
+    first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
+    print(f"    [dim]First URL: {first_url}[/dim]")
+
+    # 3. The CrawlMachine will create Snapshots from all URLs when started
+    #    Parser extractors run on snapshots and discover more URLs
+    #    Discovered URLs become child Snapshots (depth+1)
+
+    if index_only:
+        # Just create the crawl but don't start processing
+        print("[yellow]\\[*] Index-only mode - crawl created but not started[/yellow]")
+        # Create snapshots for all URLs in the crawl
+        for url in crawl.get_urls_list():
+            snapshot, _ = Snapshot.objects.update_or_create(
+                crawl=crawl,
+                url=url,
+                defaults={
+                    "status": Snapshot.INITIAL_STATE,
+                    "retry_at": timezone.now(),
+                    "timestamp": str(timezone.now().timestamp()),
+                    "depth": 0,
+                },
+            )
+            if tag:
+                snapshot.save_tags(tag.split(","))
+            snapshot.ensure_crawl_symlink()
+        return crawl, crawl.snapshot_set.all()
+
+    if bg:
+        crawl.create_snapshots_from_urls()
+
+    # 5. Start the crawl runner to process the queue
+    #    The runner will:
+    #    - Process Crawl -> create Snapshots from all URLs
+    #    - Process Snapshots -> run extractors
+    #    - Parser extractors discover new URLs -> create child Snapshots
+    #    - Repeat until max_depth reached
+
+    if bg:
+        # Background mode: just queue work and return (background runner via server will pick it up)
+        print(
+            "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
+        )
+    else:
+        # Foreground mode: run full crawl runner until all work is done
+        print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
+        run_crawl(str(crawl.id))
+
+        # Print summary for foreground runs
+        try:
+            crawl.refresh_from_db()
+            snapshots_count = crawl.snapshot_set.count()
+            try:
+                from django.db.models import Count, Sum
+
+                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("archiveresult__output_size"))
+                total_bytes = int(totals["total_bytes"] or 0) if totals["snapshot_count"] else 0
+            except Exception:
+                total_bytes, _, _ = get_dir_size(crawl.output_dir)
+            total_size = printable_filesize(total_bytes)
+            total_time = timezone.now() - started_at
+            total_seconds = int(total_time.total_seconds())
+            mins, secs = divmod(total_seconds, 60)
+            hours, mins = divmod(mins, 60)
+            if hours:
+                duration_str = f"{hours}h {mins}m {secs}s"
+            elif mins:
+                duration_str = f"{mins}m {secs}s"
+            else:
+                duration_str = f"{secs}s"
+
+            # Output dir relative to DATA_DIR
+            try:
+                rel_output = Path(crawl.output_dir).relative_to(CONSTANTS.DATA_DIR)
+                rel_output_str = f"./{rel_output}"
+            except Exception:
+                rel_output_str = str(crawl.output_dir)
+
+            bind_addr = SERVER_CONFIG.BIND_ADDR or "127.0.0.1:8000"
+            if bind_addr.startswith("http://") or bind_addr.startswith("https://"):
+                base_url = bind_addr
+            else:
+                base_url = f"http://{bind_addr}"
+            admin_url = f"{base_url}/admin/crawls/crawl/{crawl.id}/change/"
+
+            print("\n[bold]crawl output saved to:[/bold]")
+            print(f"  {rel_output_str}")
+            print(f"  {admin_url}")
+            print(f"\n[bold]total urls snapshotted:[/bold] {snapshots_count}")
+            print(f"[bold]total size:[/bold] {total_size}")
+            print(f"[bold]total time:[/bold] {duration_str}")
+        except Exception:
+            # Summary is best-effort; avoid failing the command if something goes wrong
+            pass
+
+    # 6. Return the list of Snapshots in this crawl
+    snapshots = crawl.snapshot_set.all()
+    return crawl, snapshots
+
+
+@click.command()
+@click.option(
+    "--depth",
+    "-d",
+    type=click.Choice([str(i) for i in range(5)]),
+    default="0",
+    help="Recursively archive linked pages up to N hops away",
+)
+@click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)")
+@click.option("--max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
+@click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3")
+@click.option("--url-allowlist", "--domain-allowlist", default="", help="Comma-separated URL/domain allowlist for this crawl")
+@click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
+@click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
+@click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
+@click.option("--persona", default="Default", help="Authentication profile to use when archiving")
+@click.option("--overwrite", "-F", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
+@click.option("--update", is_flag=True, default=None, help="Retry any previously skipped/failed URLs when re-adding them")
+@click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
+@click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
+@click.argument("urls", nargs=-1, type=click.Path())
@docstring(add.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=add.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        '--tag', '-t',
-        type=str,
-        default='',
-        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
-    )
-    parser.add_argument(
-        '--update', #'-u',
-        action='store_true',
-        default=not ARCHIVING_CONFIG.ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
-        help="Also retry previously skipped/failed links when adding new links",
-    )
-    parser.add_argument(
-        '--update-all', #'-n',
-        action='store_true',
-        default=False, 
-        help="Also update ALL links in index when finished adding new links",
-    )
-    parser.add_argument(
-        '--index-only', #'-o',
-        action='store_true',
-        help="Add the links to the main index without archiving them",
-    )
-    parser.add_argument(
-        'urls',
-        nargs='*',
-        type=str,
-        default=None,
-        help=(
-            'URLs or paths to archive e.g.:\n'
-            '    https://getpocket.com/users/USERNAME/feed/all\n'
-            '    https://example.com/some/rss/feed.xml\n'
-            '    https://example.com\n'
-            '    ~/Downloads/firefox_bookmarks_export.html\n'
-            '    ~/Desktop/sites_list.csv\n'
-        )
-    )
-    parser.add_argument(
-        "--depth",
-        action="store",
-        default=0,
-        choices=[0, 1],
-        type=int,
-        help="Recursively archive all linked pages up to this many hops away"
-    )
-    parser.add_argument(
-        "--overwrite",
-        default=False,
-        action="store_true",
-        help="Re-archive URLs from scratch, overwriting any existing files"
-    )
-    parser.add_argument(
-        "--init", #'-i',
-        action='store_true',
-        help="Init/upgrade the curent data directory before adding",
-    )
-    parser.add_argument(
-        "--extract",
-        type=str,
-        help="Pass a list of the extractors to be used. If the method name is not correct, it will be ignored. \
-              This does not take precedence over the configuration",
-        default=""
-    )
-    parser.add_argument(
-        "--parser",
-        type=str,
-        help="Parser used to read inputted URLs.",
-        default="auto",
-        choices=["auto", *PARSERS.keys()],
-    )
-    command = parser.parse_args(args or ())
-    urls = command.urls
+def main(**kwargs):
+    """Add a new URL or list of URLs to your archive"""

-    stdin_urls = ''
+    raw_urls = kwargs.pop("urls")
+    urls = _collect_input_urls(raw_urls)
    if not urls:
-        stdin_urls = accept_stdin(stdin)
+        raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
+    if int(kwargs.get("max_urls") or 0) < 0:
+        raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
+    try:
+        kwargs["max_size"] = parse_filesize_to_bytes(kwargs.get("max_size"))
+    except ValueError as err:
+        raise click.BadParameter(str(err), param_hint="--max-size") from err

-    if (stdin_urls and urls) or (not stdin and not urls):
-        stderr(
-            '[X] You must pass URLs/paths to add via stdin or CLI arguments.\n',
-            color='red',
-        )
-        raise SystemExit(2)
-    add(
-        urls=stdin_urls or urls,
-        depth=command.depth,
-        tag=command.tag,
-        update=command.update,
-        update_all=command.update_all,
-        index_only=command.index_only,
-        overwrite=command.overwrite,
-        init=command.init,
-        extractors=command.extract,
-        parser=command.parser,
-        out_dir=pwd or DATA_DIR,
-    )
+    add(urls=urls, **kwargs)


-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
-
-
-# TODO: Implement these
-#
-# parser.add_argument(
-#     '--mirror', #'-m',
-#     action='store_true',
-#     help='Archive an entire site (finding all linked pages below it on the same domain)',
-# )
-# parser.add_argument(
-#     '--crawler', #'-r',
-#     choices=('depth_first', 'breadth_first'),
-#     help='Controls which crawler to use in order to find outlinks in a given page',
-#     default=None,
-# )
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,387 @@
+#!/usr/bin/env python3
+
+"""
+archivebox archiveresult <action> [args...] [--filters]
+
+Manage ArchiveResult records (plugin extraction results).
+
+Actions:
+    create  - Create ArchiveResults for Snapshots (queue extractions)
+    list    - List ArchiveResults as JSONL (with optional filters)
+    update  - Update ArchiveResults from stdin JSONL
+    delete  - Delete ArchiveResults from stdin JSONL
+
+Examples:
+    # Create ArchiveResults for snapshots (queue for extraction)
+    archivebox snapshot list --status=queued | archivebox archiveresult create
+    archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>
+
+    # List with filters
+    archivebox archiveresult list --status=failed
+    archivebox archiveresult list --plugin=screenshot --status=succeeded
+
+    # Update (reset failed extractions to queued)
+    archivebox archiveresult list --status=failed | archivebox archiveresult update --status=queued
+
+    # Delete
+    archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes
+
+    # Re-run failed extractions
+    archivebox archiveresult list --status=failed | archivebox run
+"""
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox archiveresult"
+
+import sys
+
+import rich_click as click
+from rich import print as rprint
+
+from archivebox.cli.cli_utils import apply_filters
+
+
+def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict:
+    return {
+        "type": "ArchiveResult",
+        "snapshot_id": str(snapshot_id),
+        "plugin": plugin,
+        "hook_name": hook_name,
+        "status": status,
+    }
+
+
+# =============================================================================
+# CREATE
+# =============================================================================
+
+
+def create_archiveresults(
+    snapshot_id: str | None = None,
+    plugin: str | None = None,
+    status: str = "queued",
+) -> int:
+    """
+    Create ArchiveResult request records for Snapshots.
+
+    Reads Snapshot records from stdin and emits ArchiveResult request JSONL.
+    Pass-through: Non-Snapshot/ArchiveResult records are output unchanged.
+    If --plugin is specified, only emits requests for that plugin.
+    Otherwise, emits requests for all enabled snapshot hooks.
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
+    from archivebox.config.configset import get_config
+    from archivebox.hooks import discover_hooks
+    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
+    from archivebox.core.models import Snapshot
+
+    is_tty = sys.stdout.isatty()
+
+    # If snapshot_id provided directly, use that
+    if snapshot_id:
+        try:
+            snapshots = [Snapshot.objects.get(id=snapshot_id)]
+            pass_through_records = []
+        except Snapshot.DoesNotExist:
+            rprint(f"[red]Snapshot not found: {snapshot_id}[/red]", file=sys.stderr)
+            return 1
+    else:
+        # Read from stdin
+        records = list(read_stdin())
+        if not records:
+            rprint("[yellow]No Snapshot records provided via stdin[/yellow]", file=sys.stderr)
+            return 1
+
+        # Separate snapshot records from pass-through records
+        snapshot_ids = []
+        pass_through_records = []
+
+        for record in records:
+            record_type = record.get("type", "")
+
+            if record_type == TYPE_SNAPSHOT:
+                # Pass through the Snapshot record itself
+                pass_through_records.append(record)
+                if record.get("id"):
+                    snapshot_ids.append(record["id"])
+
+            elif record_type == TYPE_ARCHIVERESULT:
+                # ArchiveResult records: pass through if they have an id
+                if record.get("id"):
+                    pass_through_records.append(record)
+                # If no id, we could create it, but for now just pass through
+                else:
+                    pass_through_records.append(record)
+
+            elif record_type:
+                # Other typed records (Crawl, Tag, etc): pass through
+                pass_through_records.append(record)
+
+            elif record.get("id"):
+                # Untyped record with id - assume it's a snapshot ID
+                snapshot_ids.append(record["id"])
+
+        # Output pass-through records first
+        if not is_tty:
+            for record in pass_through_records:
+                write_record(record)
+
+        if not snapshot_ids:
+            if pass_through_records:
+                rprint(f"[dim]Passed through {len(pass_through_records)} records, no new snapshots to process[/dim]", file=sys.stderr)
+                return 0
+            rprint("[yellow]No valid Snapshot IDs in input[/yellow]", file=sys.stderr)
+            return 1
+
+        snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids))
+
+    if not snapshots:
+        rprint("[yellow]No matching snapshots found[/yellow]", file=sys.stderr)
+        return 0 if pass_through_records else 1
+
+    created_count = 0
+    for snapshot in snapshots:
+        if plugin:
+            if not is_tty:
+                write_record(build_archiveresult_request(snapshot.id, plugin, status=status))
+            created_count += 1
+        else:
+            config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
+            hooks = discover_hooks("Snapshot", config=config)
+            for hook_path in hooks:
+                hook_name = hook_path.name
+                plugin_name = hook_path.parent.name
+                if not is_tty:
+                    write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
+                created_count += 1
+
+    rprint(f"[green]Created {created_count} archive result request records[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+
+def list_archiveresults(
+    status: str | None = None,
+    plugin: str | None = None,
+    snapshot_id: str | None = None,
+    limit: int | None = None,
+) -> int:
+    """
+    List ArchiveResults as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.core.models import ArchiveResult
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = ArchiveResult.objects.all().order_by("-start_ts")
+
+    # Apply filters
+    filter_kwargs = {
+        "status": status,
+        "plugin": plugin,
+        "snapshot_id": snapshot_id,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for result in queryset:
+        if is_tty:
+            status_color = {
+                "queued": "yellow",
+                "started": "blue",
+                "succeeded": "green",
+                "failed": "red",
+                "skipped": "dim",
+                "noresults": "dim",
+                "backoff": "magenta",
+            }.get(result.status, "dim")
+            rprint(
+                f"[{status_color}]{result.status:10}[/{status_color}] {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
+            )
+        else:
+            write_record(result.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} archive results[/dim]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# UPDATE
+# =============================================================================
+
+
+def update_archiveresults(
+    status: str | None = None,
+) -> int:
+    """
+    Update ArchiveResults from stdin JSONL.
+
+    Reads ArchiveResult records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from archivebox.misc.jsonl import read_stdin, write_record
+    from archivebox.core.models import ArchiveResult
+
+    is_tty = sys.stdout.isatty()
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        result_id = record.get("id")
+        if not result_id:
+            continue
+
+        try:
+            result = ArchiveResult.objects.get(id=result_id)
+
+            # Apply updates from CLI flags
+            if status:
+                result.status = status
+
+            result.save()
+            updated_count += 1
+
+            if not is_tty:
+                write_record(result.to_json())
+
+        except ArchiveResult.DoesNotExist:
+            rprint(f"[yellow]ArchiveResult not found: {result_id}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} archive results[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# DELETE
+# =============================================================================
+
+
+def delete_archiveresults(yes: bool = False, dry_run: bool = False) -> int:
+    """
+    Delete ArchiveResults from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
+    from archivebox.core.models import ArchiveResult
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    result_ids = [r.get("id") for r in records if r.get("id")]
+
+    if not result_ids:
+        rprint("[yellow]No valid archive result IDs in input[/yellow]", file=sys.stderr)
+        return 1
+
+    results = ArchiveResult.objects.filter(id__in=result_ids)
+    count = results.count()
+
+    if count == 0:
+        rprint("[yellow]No matching archive results found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} archive results (dry run)[/yellow]", file=sys.stderr)
+        for result in results[:10]:
+            rprint(f"  [dim]{result.id}[/dim] {result.plugin} {result.snapshot.url[:40]}", file=sys.stderr)
+        if count > 10:
+            rprint(f"  ... and {count - 10} more", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = results.delete()
+    rprint(f"[green]Deleted {deleted_count} archive results[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+
+@click.group()
+def main():
+    """Manage ArchiveResult records (plugin extraction results)."""
+    pass
+
+
+@main.command("create")
+@click.option("--snapshot-id", help="Snapshot ID to create results for")
+@click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
+@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
+def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
+    """Create ArchiveResults for Snapshots from stdin JSONL."""
+    sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
+
+
+@main.command("list")
+@click.option("--status", "-s", help="Filter by status (queued, started, succeeded, failed, skipped)")
+@click.option("--plugin", "-p", help="Filter by plugin name")
+@click.option("--snapshot-id", help="Filter by snapshot ID")
+@click.option("--limit", "-n", type=int, help="Limit number of results")
+def list_cmd(
+    status: str | None,
+    plugin: str | None,
+    snapshot_id: str | None,
+    limit: int | None,
+):
+    """List ArchiveResults as JSONL."""
+    sys.exit(
+        list_archiveresults(
+            status=status,
+            plugin=plugin,
+            snapshot_id=snapshot_id,
+            limit=limit,
+        ),
+    )
+
+
+@main.command("update")
+@click.option("--status", "-s", help="Set status")
+def update_cmd(status: str | None):
+    """Update ArchiveResults from stdin JSONL."""
+    sys.exit(update_archiveresults(status=status))
+
+
+@main.command("delete")
+@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
+@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
+def delete_cmd(yes: bool, dry_run: bool):
+    """Delete ArchiveResults from stdin JSONL."""
+    sys.exit(delete_archiveresults(yes=yes, dry_run=dry_run))
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,319 @@
+#!/usr/bin/env python3
+
+"""
+archivebox binary <action> [args...] [--filters]
+
+Manage Binary records (detected executables like chrome, wget, etc.).
+
+Actions:
+    create  - Create/register a Binary
+    list    - List Binaries as JSONL (with optional filters)
+    update  - Update Binaries from stdin JSONL
+    delete  - Delete Binaries from stdin JSONL
+
+Examples:
+    # List all binaries
+    archivebox binary list
+
+    # List specific binary
+    archivebox binary list --name=chrome
+
+    # List binaries with specific version
+    archivebox binary list --version__icontains=120
+
+    # Delete old binary entries
+    archivebox binary list --name=chrome | archivebox binary delete --yes
+"""
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox binary"
+
+import sys
+
+import rich_click as click
+from rich import print as rprint
+
+from archivebox.cli.cli_utils import apply_filters
+
+
+# =============================================================================
+# CREATE
+# =============================================================================
+
+
+def create_binary(
+    name: str,
+    abspath: str,
+    version: str = "",
+) -> int:
+    """
+    Create/register a Binary.
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.machine.models import Binary
+
+    is_tty = sys.stdout.isatty()
+
+    if not name or not abspath:
+        rprint("[red]Both --name and --abspath are required[/red]", file=sys.stderr)
+        return 1
+
+    try:
+        from archivebox.machine.models import Machine
+
+        machine = Machine.current()
+        created = not Binary.objects.filter(
+            machine=machine,
+            name=name,
+            abspath=abspath,
+            version=version,
+        ).exists()
+
+        # Mirror the Binary model lifecycle used elsewhere in the system so CLI
+        # records are owned by the current machine and can be safely piped into
+        # `archivebox run` without creating invalid rows missing machine_id.
+        binary = Binary.from_json(
+            {
+                "name": name,
+                "abspath": abspath,
+                "version": version,
+                "binproviders": "env",
+                "binprovider": "env",
+            },
+        )
+        if binary is None:
+            raise ValueError("failed to create binary record")
+
+        if not is_tty:
+            write_record(binary.to_json())
+
+        if created:
+            rprint(f"[green]Created binary: {name} at {abspath}[/green]", file=sys.stderr)
+        else:
+            rprint(f"[dim]Binary already exists: {name} at {abspath}[/dim]", file=sys.stderr)
+
+        return 0
+
+    except Exception as e:
+        rprint(f"[red]Error creating binary: {e}[/red]", file=sys.stderr)
+        return 1
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+
+def list_binaries(
+    name: str | None = None,
+    abspath__icontains: str | None = None,
+    version__icontains: str | None = None,
+    limit: int | None = None,
+) -> int:
+    """
+    List Binaries as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.machine.models import Binary
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Binary.objects.all().order_by("name", "-modified_at", "-created_at")
+
+    # Apply filters
+    filter_kwargs = {
+        "name": name,
+        "abspath__icontains": abspath__icontains,
+        "version__icontains": version__icontains,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for binary in queryset:
+        if is_tty:
+            rprint(f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}")
+        else:
+            write_record(binary.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} binaries[/dim]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# UPDATE
+# =============================================================================
+
+
+def update_binaries(
+    version: str | None = None,
+    abspath: str | None = None,
+) -> int:
+    """
+    Update Binaries from stdin JSONL.
+
+    Reads Binary records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from archivebox.misc.jsonl import read_stdin, write_record
+    from archivebox.machine.models import Binary
+
+    is_tty = sys.stdout.isatty()
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        binary_id = record.get("id")
+        if not binary_id:
+            continue
+
+        try:
+            binary = Binary.objects.get(id=binary_id)
+
+            # Apply updates from CLI flags
+            if version:
+                binary.version = version
+            if abspath:
+                binary.abspath = abspath
+
+            binary.save()
+            updated_count += 1
+
+            if not is_tty:
+                write_record(binary.to_json())
+
+        except Binary.DoesNotExist:
+            rprint(f"[yellow]Binary not found: {binary_id}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} binaries[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# DELETE
+# =============================================================================
+
+
+def delete_binaries(yes: bool = False, dry_run: bool = False) -> int:
+    """
+    Delete Binaries from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
+    from archivebox.machine.models import Binary
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    binary_ids = [r.get("id") for r in records if r.get("id")]
+
+    if not binary_ids:
+        rprint("[yellow]No valid binary IDs in input[/yellow]", file=sys.stderr)
+        return 1
+
+    binaries = Binary.objects.filter(id__in=binary_ids)
+    count = binaries.count()
+
+    if count == 0:
+        rprint("[yellow]No matching binaries found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} binaries (dry run)[/yellow]", file=sys.stderr)
+        for binary in binaries:
+            rprint(f"  {binary.name} {binary.abspath}", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = binaries.delete()
+    rprint(f"[green]Deleted {deleted_count} binaries[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+
+@click.group()
+def main():
+    """Manage Binary records (detected executables)."""
+    pass
+
+
+@main.command("create")
+@click.option("--name", "-n", required=True, help="Binary name (e.g., chrome, wget)")
+@click.option("--abspath", "-p", required=True, help="Absolute path to binary")
+@click.option("--version", "-v", default="", help="Binary version")
+def create_cmd(name: str, abspath: str, version: str):
+    """Create/register a Binary."""
+    sys.exit(create_binary(name=name, abspath=abspath, version=version))
+
+
+@main.command("list")
+@click.option("--name", "-n", help="Filter by name")
+@click.option("--abspath__icontains", help="Filter by path contains")
+@click.option("--version__icontains", help="Filter by version contains")
+@click.option("--limit", type=int, help="Limit number of results")
+def list_cmd(
+    name: str | None,
+    abspath__icontains: str | None,
+    version__icontains: str | None,
+    limit: int | None,
+):
+    """List Binaries as JSONL."""
+    sys.exit(
+        list_binaries(
+            name=name,
+            abspath__icontains=abspath__icontains,
+            version__icontains=version__icontains,
+            limit=limit,
+        ),
+    )
+
+
+@main.command("update")
+@click.option("--version", "-v", help="Set version")
+@click.option("--abspath", "-p", help="Set path")
+def update_cmd(version: str | None, abspath: str | None):
+    """Update Binaries from stdin JSONL."""
+    sys.exit(update_binaries(version=version, abspath=abspath))
+
+
+@main.command("delete")
+@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
+@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
+def delete_cmd(yes: bool, dry_run: bool):
+    """Delete Binaries from stdin JSONL."""
+    sys.exit(delete_binaries(yes=yes, dry_run=dry_run))
+
+
+if __name__ == "__main__":
+    main()
@@ -1,71 +1,177 @@
 #!/usr/bin/env python3

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox config'
+__package__ = "archivebox.cli"

 import sys
-import argparse
-from pathlib import Path
+import rich_click as click
+from rich import print
+from benedict import benedict

-from typing import Optional, List, IO
-
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..main import config
-from ..logging_util import SmartFormatter, accept_stdin
+from archivebox.misc.util import docstring, enforce_types
+from archivebox.misc.toml_util import CustomTOMLEncoder


+@enforce_types
+def config(
+    *keys,
+    get: bool = False,
+    set: bool = False,
+    search: bool = False,
+    reset: bool = False,
+    **kwargs,
+) -> None:
+    """Get and set your ArchiveBox project configuration values"""
+
+    from archivebox.misc.checks import check_data_folder
+    from archivebox.misc.logging_util import printable_config
+    from archivebox.config.collection import load_all_config, write_config_file, get_real_name
+    from archivebox.config.configset import get_flat_config, get_all_configs
+
+    check_data_folder()
+
+    FLAT_CONFIG = get_flat_config()
+    CONFIGS = get_all_configs()
+
+    config_options: list[str] = list(kwargs.pop("key=value", []) or keys or [f"{key}={val}" for key, val in kwargs.items()])
+    no_args = not (get or set or reset or config_options)
+
+    matching_config = {}
+    if search:
+        if config_options:
+            config_options = [get_real_name(key) for key in config_options]
+            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            for config_section in CONFIGS.values():
+                aliases = getattr(config_section, "aliases", {})
+
+                for search_key in config_options:
+                    # search all aliases in the section
+                    for alias_key, key in aliases.items():
+                        if search_key.lower() in alias_key.lower():
+                            matching_config[key] = dict(config_section)[key]
+
+                    # search all keys and values in the section
+                    for existing_key, value in dict(config_section).items():
+                        if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
+                            matching_config[existing_key] = value
+
+        print(printable_config(matching_config))
+        raise SystemExit(not matching_config)
+
+    elif get or no_args:
+        if config_options:
+            config_options = [get_real_name(key) for key in config_options]
+            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
+            if failed_config:
+                print("\n[red][X] These options failed to get[/red]")
+                print("    {}".format("\n    ".join(config_options)))
+                raise SystemExit(1)
+        else:
+            matching_config = FLAT_CONFIG
+
+        # Display core config sections
+        for config_section in CONFIGS.values():
+            section_header = getattr(config_section, "toml_section_header", "")
+            if isinstance(section_header, str) and section_header:
+                print(f"[grey53]\\[{section_header}][/grey53]")
+            else:
+                print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
+
+            kv_in_section = {key: val for key, val in dict(config_section).items() if key in matching_config}
+            print(benedict(kv_in_section).to_toml(encoder=CustomTOMLEncoder()).strip().replace("\n\n", "\n"))
+            print("[grey53]################################################################[/grey53]")
+
+        # Display plugin config section
+        from archivebox.hooks import discover_plugin_configs
+
+        plugin_configs = discover_plugin_configs()
+        plugin_keys = {}
+
+        # Collect all plugin config keys
+        for plugin_name, schema in plugin_configs.items():
+            if "properties" not in schema:
+                continue
+            for key in schema["properties"].keys():
+                if key in matching_config:
+                    plugin_keys[key] = matching_config[key]
+
+        # Display all plugin config in single [PLUGINS] section
+        if plugin_keys:
+            print("[grey53]\\[PLUGINS][/grey53]")
+            print(benedict(plugin_keys).to_toml(encoder=CustomTOMLEncoder()).strip().replace("\n\n", "\n"))
+            print("[grey53]################################################################[/grey53]")
+
+        raise SystemExit(not matching_config)
+
+    elif set:
+        new_config = {}
+        failed_options = []
+        for line in config_options:
+            if line.startswith("#") or not line.strip():
+                continue
+            if "=" not in line:
+                print("[red][X] Config KEY=VALUE must have an = sign in it[/red]")
+                print(f"    {line}")
+                raise SystemExit(2)
+
+            raw_key, val = line.split("=", 1)
+            raw_key = raw_key.upper().strip()
+            key = get_real_name(raw_key)
+            if key != raw_key:
+                print(
+                    f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
+                )
+
+            if key in FLAT_CONFIG:
+                new_config[key] = val.strip()
+            else:
+                failed_options.append(line)
+
+        if new_config:
+            before = FLAT_CONFIG
+            matching_config = write_config_file(new_config)
+            after = {**load_all_config(), **get_flat_config()}
+            print(printable_config(matching_config))
+
+            side_effect_changes = {}
+            for key, val in after.items():
+                if key in FLAT_CONFIG and (str(before[key]) != str(after[key])) and (key not in matching_config):
+                    side_effect_changes[key] = after[key]
+
+            if side_effect_changes:
+                print(file=sys.stderr)
+                print("[yellow][i] Note: This change also affected these other options that depended on it:[/yellow]", file=sys.stderr)
+                print("    {}".format(printable_config(side_effect_changes, prefix="    ")), file=sys.stderr)
+
+        if failed_options:
+            print()
+            print("[red][X] These options failed to set (check for typos):[/red]")
+            print("    {}".format("\n    ".join(failed_options)))
+            raise SystemExit(1)
+
+    elif reset:
+        print("[red][X] This command is not implemented yet.[/red]")
+        print("    Please manually remove the relevant lines from your config file:")
+        raise SystemExit(2)
+
+    else:
+        print("[red][X] You must pass either --get or --set, or no arguments to get the whole config.[/red]")
+        print("    archivebox config")
+        print("    archivebox config --get SOME_KEY")
+        print("    archivebox config --set SOME_KEY=SOME_VALUE")
+        raise SystemExit(2)
+
+
+@click.command()
+@click.option("--search", is_flag=True, help="Search config KEYs, VALUEs, and ALIASES for the given term")
+@click.option("--get", is_flag=True, help="Get the value for the given config KEYs")
+@click.option("--set", is_flag=True, help="Set the given KEY=VALUE config values")
+@click.option("--reset", is_flag=True, help="Reset the given KEY config values to their defaults")
+@click.argument("KEY=VALUE", nargs=-1, type=str)
@docstring(config.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=config.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    group = parser.add_mutually_exclusive_group()
-    parser.add_argument(
-        '--search',
-        action='store_true',
-        help="Search config KEYs, VALUEs, and ALIASES for the given term",
-    )
-    group.add_argument(
-        '--get', #'-g',
-        action='store_true',
-        help="Get the value for the given config KEYs",
-    )
-    group.add_argument(
-        '--set', #'-s',
-        action='store_true',
-        help="Set the given KEY=VALUE config values",
-    )
-    group.add_argument(
-        '--reset', #'-s',
-        action='store_true',
-        help="Reset the given KEY config values to their defaults",
-    )
-    parser.add_argument(
-        'config_options',
-        nargs='*',
-        type=str,
-        help='KEY or KEY=VALUE formatted config values to get or set',
-    )
-    command = parser.parse_args(args or ())
-
-    config_options_str = ''
-    if not command.config_options:
-        config_options_str = accept_stdin(stdin)
-
-    config(
-        config_options_str=config_options_str,
-        config_options=command.config_options,
-        search=command.search,
-        get=command.get,
-        set=command.set,
-        reset=command.reset,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
-    )
+def main(**kwargs) -> None:
+    config(**kwargs)


-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,384 @@
+#!/usr/bin/env python3
+
+"""
+archivebox crawl <action> [args...] [--filters]
+
+Manage Crawl records.
+
+Actions:
+    create  - Create Crawl jobs from URLs
+    list    - List Crawls as JSONL (with optional filters)
+    update  - Update Crawls from stdin JSONL
+    delete  - Delete Crawls from stdin JSONL
+
+Examples:
+    # Create
+    archivebox crawl create https://example.com https://foo.com --depth=1
+    archivebox crawl create --tag=news https://example.com
+
+    # List with filters
+    archivebox crawl list --status=queued
+    archivebox crawl list --urls__icontains=example.com
+
+    # Update
+    archivebox crawl list --status=started | archivebox crawl update --status=queued
+
+    # Delete
+    archivebox crawl list --urls__icontains=spam.com | archivebox crawl delete --yes
+
+    # Full pipeline
+    archivebox crawl create https://example.com | archivebox snapshot create | archivebox run
+"""
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox crawl"
+
+import sys
+from collections.abc import Iterable
+
+import rich_click as click
+from rich import print as rprint
+
+from archivebox.cli.cli_utils import apply_filters
+
+
+# =============================================================================
+# CREATE
+# =============================================================================
+
+
+def create_crawl(
+    urls: Iterable[str],
+    depth: int = 0,
+    tag: str = "",
+    status: str = "queued",
+    created_by_id: int | None = None,
+) -> int:
+    """
+    Create a Crawl job from URLs.
+
+    Takes URLs as args or stdin, creates one Crawl with all URLs, outputs JSONL.
+    Pass-through: Records that are not URLs are output unchanged (for piping).
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
+    from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl
+
+    created_by_id = created_by_id or get_or_create_system_user_pk()
+    is_tty = sys.stdout.isatty()
+
+    # Collect all input records
+    records = list(read_args_or_stdin(urls))
+
+    if not records:
+        rprint("[yellow]No URLs provided. Pass URLs as arguments or via stdin.[/yellow]", file=sys.stderr)
+        return 1
+
+    # Separate pass-through records from URL records
+    url_list = []
+    pass_through_records = []
+
+    for record in records:
+        record_type = record.get("type", "")
+
+        # Pass-through: output records that aren't URL/Crawl types
+        if record_type and record_type != TYPE_CRAWL and not record.get("url") and not record.get("urls"):
+            pass_through_records.append(record)
+            continue
+
+        # Handle existing Crawl records (just pass through with id)
+        if record_type == TYPE_CRAWL and record.get("id"):
+            pass_through_records.append(record)
+            continue
+
+        # Collect URLs
+        url = record.get("url")
+        if url:
+            url_list.append(url)
+
+        # Handle 'urls' field (newline-separated)
+        urls_field = record.get("urls")
+        if urls_field:
+            for line in urls_field.split("\n"):
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    url_list.append(line)
+
+    # Output pass-through records first
+    if not is_tty:
+        for record in pass_through_records:
+            write_record(record)
+
+    if not url_list:
+        if pass_through_records:
+            # If we had pass-through records but no URLs, that's OK
+            rprint(f"[dim]Passed through {len(pass_through_records)} records, no new URLs[/dim]", file=sys.stderr)
+            return 0
+        rprint("[red]No valid URLs found[/red]", file=sys.stderr)
+        return 1
+
+    try:
+        # Build crawl record with all URLs as newline-separated string
+        crawl_record = {
+            "urls": "\n".join(url_list),
+            "max_depth": depth,
+            "tags_str": tag,
+            "status": status,
+            "label": "",
+        }
+
+        crawl = Crawl.from_json(crawl_record, overrides={"created_by_id": created_by_id})
+        if not crawl:
+            rprint("[red]Failed to create crawl[/red]", file=sys.stderr)
+            return 1
+
+        # Output JSONL record (only when piped)
+        if not is_tty:
+            write_record(crawl.to_json())
+
+        rprint(f"[green]Created crawl with {len(url_list)} URLs[/green]", file=sys.stderr)
+
+        # If TTY, show human-readable output
+        if is_tty:
+            rprint(f"  [dim]{crawl.id}[/dim]", file=sys.stderr)
+            for url in url_list[:5]:  # Show first 5 URLs
+                rprint(f"    {url[:70]}", file=sys.stderr)
+            if len(url_list) > 5:
+                rprint(f"    ... and {len(url_list) - 5} more", file=sys.stderr)
+
+        return 0
+
+    except Exception as e:
+        rprint(f"[red]Error creating crawl: {e}[/red]", file=sys.stderr)
+        return 1
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+
+def list_crawls(
+    status: str | None = None,
+    urls__icontains: str | None = None,
+    max_depth: int | None = None,
+    limit: int | None = None,
+) -> int:
+    """
+    List Crawls as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.crawls.models import Crawl
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Crawl.objects.all().order_by("-created_at")
+
+    # Apply filters
+    filter_kwargs = {
+        "status": status,
+        "urls__icontains": urls__icontains,
+        "max_depth": max_depth,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for crawl in queryset:
+        if is_tty:
+            status_color = {
+                "queued": "yellow",
+                "started": "blue",
+                "sealed": "green",
+            }.get(crawl.status, "dim")
+            url_preview = crawl.urls[:50].replace("\n", " ")
+            rprint(f"[{status_color}]{crawl.status:8}[/{status_color}] [dim]{crawl.id}[/dim] {url_preview}...")
+        else:
+            write_record(crawl.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} crawls[/dim]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# UPDATE
+# =============================================================================
+
+
+def update_crawls(
+    status: str | None = None,
+    max_depth: int | None = None,
+) -> int:
+    """
+    Update Crawls from stdin JSONL.
+
+    Reads Crawl records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from django.utils import timezone
+
+    from archivebox.misc.jsonl import read_stdin, write_record
+    from archivebox.crawls.models import Crawl
+
+    is_tty = sys.stdout.isatty()
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        crawl_id = record.get("id")
+        if not crawl_id:
+            continue
+
+        try:
+            crawl = Crawl.objects.get(id=crawl_id)
+
+            # Apply updates from CLI flags
+            if status:
+                crawl.status = status
+                crawl.retry_at = timezone.now()
+            if max_depth is not None:
+                crawl.max_depth = max_depth
+
+            crawl.save()
+            updated_count += 1
+
+            if not is_tty:
+                write_record(crawl.to_json())
+
+        except Crawl.DoesNotExist:
+            rprint(f"[yellow]Crawl not found: {crawl_id}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} crawls[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# DELETE
+# =============================================================================
+
+
+def delete_crawls(yes: bool = False, dry_run: bool = False) -> int:
+    """
+    Delete Crawls from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
+    from archivebox.crawls.models import Crawl
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    crawl_ids = [r.get("id") for r in records if r.get("id")]
+
+    if not crawl_ids:
+        rprint("[yellow]No valid crawl IDs in input[/yellow]", file=sys.stderr)
+        return 1
+
+    crawls = Crawl.objects.filter(id__in=crawl_ids)
+    count = crawls.count()
+
+    if count == 0:
+        rprint("[yellow]No matching crawls found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} crawls (dry run)[/yellow]", file=sys.stderr)
+        for crawl in crawls:
+            url_preview = crawl.urls[:50].replace("\n", " ")
+            rprint(f"  [dim]{crawl.id}[/dim] {url_preview}...", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = crawls.delete()
+    rprint(f"[green]Deleted {deleted_count} crawls[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+
+@click.group()
+def main():
+    """Manage Crawl records."""
+    pass
+
+
+@main.command("create")
+@click.argument("urls", nargs=-1)
+@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
+@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
+@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
+def create_cmd(urls: tuple, depth: int, tag: str, status: str):
+    """Create a Crawl job from URLs or stdin."""
+    sys.exit(create_crawl(urls, depth=depth, tag=tag, status=status))
+
+
+@main.command("list")
+@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
+@click.option("--urls__icontains", help="Filter by URLs contains")
+@click.option("--max-depth", type=int, help="Filter by max depth")
+@click.option("--limit", "-n", type=int, help="Limit number of results")
+def list_cmd(
+    status: str | None,
+    urls__icontains: str | None,
+    max_depth: int | None,
+    limit: int | None,
+):
+    """List Crawls as JSONL."""
+    sys.exit(
+        list_crawls(
+            status=status,
+            urls__icontains=urls__icontains,
+            max_depth=max_depth,
+            limit=limit,
+        ),
+    )
+
+
+@main.command("update")
+@click.option("--status", "-s", help="Set status")
+@click.option("--max-depth", type=int, help="Set max depth")
+def update_cmd(status: str | None, max_depth: int | None):
+    """Update Crawls from stdin JSONL."""
+    sys.exit(update_crawls(status=status, max_depth=max_depth))
+
+
+@main.command("delete")
+@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
+@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
+def delete_cmd(yes: bool, dry_run: bool):
+    """Delete Crawls from stdin JSONL."""
+    sys.exit(delete_crawls(yes=yes, dry_run=dry_run))
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox crawl"
+
+import sys
+
+import rich_click as click
+
+from archivebox.cli.archivebox_add import add
+
+
+@click.command(context_settings={"ignore_unknown_options": True})
+@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
+@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
+@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
+@click.option("--wait/--no-wait", "wait", default=True, help="Accepted for backwards compatibility")
+@click.argument("urls", nargs=-1)
+def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]):
+    """Backwards-compatible `archivebox crawl URL...` entrypoint."""
+    del status, wait
+    add(list(urls), depth=depth, tag=tag, index_only=True, bg=True)
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+
+"""
+archivebox extract [snapshot_ids...] [--plugins=NAMES]
+
+Run plugins on Snapshots. Accepts snapshot IDs as arguments, from stdin, or via JSONL.
+
+Input formats:
+    - Snapshot UUIDs (one per line)
+    - JSONL: {"type": "Snapshot", "id": "...", "url": "..."}
+    - JSONL: {"type": "ArchiveResult", "snapshot_id": "...", "plugin": "..."}
+
+Output (JSONL):
+    {"type": "ArchiveResult", "id": "...", "snapshot_id": "...", "plugin": "...", "status": "..."}
+
+Examples:
+    # Extract specific snapshot
+    archivebox extract 01234567-89ab-cdef-0123-456789abcdef
+
+    # Pipe from snapshot command
+    archivebox snapshot https://example.com | archivebox extract
+
+    # Run specific plugins only
+    archivebox extract --plugins=screenshot,singlefile 01234567-89ab-cdef-0123-456789abcdef
+
+    # Chain commands
+    archivebox crawl https://example.com | archivebox snapshot | archivebox extract
+"""
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox extract"
+
+import sys
+from collections import defaultdict
+
+import rich_click as click
+
+
+def process_archiveresult_by_id(archiveresult_id: str) -> int:
+    """
+    Re-run extraction for a single ArchiveResult by ID.
+
+    ArchiveResults are projected status rows, not queued work items. Re-running
+    a single result means resetting that row and queueing its parent snapshot
+    through the shared crawl runner with the corresponding plugin selected.
+    """
+    from rich import print as rprint
+    from django.utils import timezone
+    from archivebox.core.models import ArchiveResult
+    from archivebox.services.runner import run_crawl
+
+    try:
+        archiveresult = ArchiveResult.objects.get(id=archiveresult_id)
+    except ArchiveResult.DoesNotExist:
+        rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
+        return 1
+
+    rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
+
+    try:
+        archiveresult.reset_for_retry()
+        snapshot = archiveresult.snapshot
+        snapshot.status = snapshot.StatusChoices.QUEUED
+        snapshot.retry_at = timezone.now()
+        snapshot.save(update_fields=["status", "retry_at", "modified_at"])
+
+        crawl = snapshot.crawl
+        if crawl.status != crawl.StatusChoices.STARTED:
+            crawl.status = crawl.StatusChoices.QUEUED
+        crawl.retry_at = timezone.now()
+        crawl.save(update_fields=["status", "retry_at", "modified_at"])
+
+        run_crawl(str(crawl.id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
+        archiveresult.refresh_from_db()
+
+        if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
+            print(f"[green]Extraction succeeded: {archiveresult.output_str}[/green]")
+            return 0
+        elif archiveresult.status == ArchiveResult.StatusChoices.NORESULTS:
+            print(f"[dim]Extraction completed with no results: {archiveresult.output_str}[/dim]")
+            return 0
+        elif archiveresult.status == ArchiveResult.StatusChoices.FAILED:
+            print(f"[red]Extraction failed: {archiveresult.output_str}[/red]", file=sys.stderr)
+            return 1
+        else:
+            # Still in progress or backoff - not a failure
+            print(f"[yellow]Extraction status: {archiveresult.status}[/yellow]")
+            return 0
+
+    except Exception as e:
+        print(f"[red]Extraction error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+        return 1
+
+
+def run_plugins(
+    args: tuple,
+    records: list[dict] | None = None,
+    plugins: str = "",
+    wait: bool = True,
+    emit_results: bool = True,
+) -> int:
+    """
+    Run plugins on Snapshots from input.
+
+    Reads Snapshot IDs or JSONL from args/stdin, runs plugins, outputs JSONL.
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
+    from rich import print as rprint
+    from django.utils import timezone
+
+    from archivebox.misc.jsonl import (
+        read_args_or_stdin,
+        write_record,
+        TYPE_SNAPSHOT,
+        TYPE_ARCHIVERESULT,
+    )
+    from archivebox.core.models import Snapshot
+    from archivebox.services.runner import run_crawl
+
+    is_tty = sys.stdout.isatty()
+
+    # Parse comma-separated plugins list once (reused in creation and filtering)
+    plugins_list = [p.strip() for p in plugins.split(",") if p.strip()] if plugins else []
+
+    # Parse stdin/args exactly once per CLI invocation.
+    # `main()` may already have consumed stdin to distinguish Snapshot input from
+    # ArchiveResult IDs; if so, it must pass the parsed records through here
+    # instead of asking this helper to reread an already-drained pipe.
+    if records is None:
+        records = list(read_args_or_stdin(args))
+
+    if not records:
+        rprint("[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]", file=sys.stderr)
+        return 1
+
+    # Gather snapshot IDs and optional plugin constraints to process
+    snapshot_ids = set()
+    requested_plugins_by_snapshot: dict[str, set[str]] = defaultdict(set)
+    for record in records:
+        record_type = record.get("type")
+
+        if record_type == TYPE_SNAPSHOT:
+            snapshot_id = record.get("id")
+            if snapshot_id:
+                snapshot_ids.add(snapshot_id)
+            elif record.get("url"):
+                # Look up by URL (get most recent if multiple exist)
+                snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first()
+                if snap:
+                    snapshot_ids.add(str(snap.id))
+                else:
+                    rprint(f"[yellow]Snapshot not found for URL: {record['url']}[/yellow]", file=sys.stderr)
+
+        elif record_type == TYPE_ARCHIVERESULT:
+            snapshot_id = record.get("snapshot_id")
+            if snapshot_id:
+                snapshot_ids.add(snapshot_id)
+                plugin_name = record.get("plugin")
+                if plugin_name and not plugins_list:
+                    requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name))
+
+        elif "id" in record:
+            # Assume it's a snapshot ID
+            snapshot_ids.add(record["id"])
+
+    if not snapshot_ids:
+        rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr)
+        return 1
+
+    # Get snapshots and ensure they have pending ArchiveResults
+    processed_count = 0
+    for snapshot_id in snapshot_ids:
+        try:
+            snapshot = Snapshot.objects.get(id=snapshot_id)
+        except Snapshot.DoesNotExist:
+            rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
+            continue
+
+        requested_plugin_names = set(plugins_list) | requested_plugins_by_snapshot.get(str(snapshot.id), set())
+        for plugin_name in requested_plugin_names:
+            existing_result = snapshot.archiveresult_set.filter(plugin=plugin_name).order_by("-created_at").first()
+            if existing_result:
+                existing_result.reset_for_retry()
+
+        # Reset snapshot status to allow processing
+        if snapshot.status == Snapshot.StatusChoices.SEALED:
+            snapshot.status = Snapshot.StatusChoices.STARTED
+            snapshot.retry_at = timezone.now()
+            snapshot.save()
+
+        processed_count += 1
+
+    if processed_count == 0:
+        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
+        return 1
+
+    rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
+
+    # Run orchestrator if --wait (default)
+    if wait:
+        rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
+        snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
+        for snapshot_id in snapshot_ids:
+            try:
+                snapshot = Snapshot.objects.only("id", "crawl_id").get(id=snapshot_id)
+            except Snapshot.DoesNotExist:
+                continue
+            snapshot_ids_by_crawl[str(snapshot.crawl_id)].add(str(snapshot.id))
+
+        for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items():
+            selected_plugins = (
+                plugins_list
+                or sorted(
+                    {plugin for snapshot_id in crawl_snapshot_ids for plugin in requested_plugins_by_snapshot.get(str(snapshot_id), set())},
+                )
+                or None
+            )
+            run_crawl(
+                crawl_id,
+                snapshot_ids=sorted(crawl_snapshot_ids),
+                selected_plugins=selected_plugins,
+            )
+
+    if not emit_results:
+        return 0
+
+    # Output results as JSONL (when piped) or human-readable (when TTY)
+    for snapshot_id in snapshot_ids:
+        try:
+            snapshot = Snapshot.objects.get(id=snapshot_id)
+            results = snapshot.archiveresult_set.all()
+            if plugins_list:
+                results = results.filter(plugin__in=plugins_list)
+
+            for result in results:
+                if is_tty:
+                    status_color = {
+                        "succeeded": "green",
+                        "failed": "red",
+                        "skipped": "yellow",
+                    }.get(result.status, "dim")
+                    rprint(
+                        f"  [{status_color}]{result.status}[/{status_color}] {result.plugin} → {result.output_str or ''}",
+                        file=sys.stderr,
+                    )
+                else:
+                    write_record(result.to_json())
+        except Snapshot.DoesNotExist:
+            continue
+
+    return 0
+
+
+def is_archiveresult_id(value: str) -> bool:
+    """Check if value looks like an ArchiveResult UUID."""
+    import re
+
+    uuid_pattern = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
+    if not uuid_pattern.match(value):
+        return False
+    # Verify it's actually an ArchiveResult (not a Snapshot or other object)
+    from archivebox.core.models import ArchiveResult
+
+    return ArchiveResult.objects.filter(id=value).exists()
+
+
+@click.command()
+@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run (e.g., screenshot,singlefile)")
+@click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")
+@click.argument("args", nargs=-1)
+def main(plugins: str, wait: bool, args: tuple):
+    """Run plugins on Snapshots, or process existing ArchiveResults by ID"""
+    from archivebox.misc.jsonl import read_args_or_stdin
+
+    # Read all input
+    records = list(read_args_or_stdin(args))
+
+    if not records:
+        from rich import print as rprint
+
+        rprint("[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]", file=sys.stderr)
+        sys.exit(1)
+
+    # Check if input looks like existing ArchiveResult IDs to process
+    all_are_archiveresult_ids = all(is_archiveresult_id(r.get("id") or r.get("url", "")) for r in records)
+
+    if all_are_archiveresult_ids:
+        # Process existing ArchiveResults by ID
+        from rich import print as rprint
+
+        exit_code = 0
+        for record in records:
+            archiveresult_id = record.get("id") or record.get("url")
+            if not isinstance(archiveresult_id, str):
+                rprint(f"[red]Invalid ArchiveResult input: {record}[/red]", file=sys.stderr)
+                exit_code = 1
+                continue
+            result = process_archiveresult_by_id(archiveresult_id)
+            if result != 0:
+                exit_code = result
+        sys.exit(exit_code)
+    else:
+        # Default behavior: run plugins on Snapshots from input
+        sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
+
+
+if __name__ == "__main__":
+    main()
@@ -1,32 +1,131 @@
 #!/usr/bin/env python3
+__package__ = "archivebox.cli"
+__command__ = "archivebox help"

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox help'
-
-import sys
-import argparse
+import os
 from pathlib import Path
-from typing import Optional, List, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..main import help
-from ..logging_util import SmartFormatter, reject_stdin
+import click
+from rich import print
+from rich.panel import Panel


-@docstring(help.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=help.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
+def help() -> None:
+    """Print the ArchiveBox help message and usage"""
+
+    from archivebox.cli import ArchiveBoxGroup
+    from archivebox.config import CONSTANTS
+    from archivebox.config.permissions import IN_DOCKER
+    from archivebox.misc.logging_util import log_cli_command
+
+    log_cli_command("help", [], None, ".")
+
+    COMMANDS_HELP_TEXT = (
+        "\n    ".join(
+            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.meta_commands.keys()
+        )
+        + "\n\n    "
+        + "\n    ".join(
+            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.setup_commands.keys()
+        )
+        + "\n\n    "
+        + "\n    ".join(
+            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.archive_commands.keys()
+        )
    )
-    parser.parse_args(args or ())
-    reject_stdin(__command__, stdin)
-    
-    help(out_dir=Path(pwd) if pwd else DATA_DIR)
+
+    DOCKER_USAGE = (
+        """
+[dodger_blue3]Docker Usage:[/dodger_blue3]
+    [grey53]# using Docker Compose:[/grey53]
+    [blue]docker compose run[/blue] [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
+
+    [grey53]# using Docker:[/grey53]
+    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 8000:8000[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
+"""
+        if IN_DOCKER
+        else ""
+    )
+    DOCKER_DOCS = (
+        "\n    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker[/link]"
+        if IN_DOCKER
+        else ""
+    )
+    DOCKER_OUTSIDE_HINT = "\n    [grey53]# outside of Docker:[/grey53]" if IN_DOCKER else ""
+    DOCKER_CMD_PREFIX = "[blue]docker ... [/blue]" if IN_DOCKER else ""
+
+    print(f"""{DOCKER_USAGE}
+[deep_sky_blue4]Usage:[/deep_sky_blue4]{DOCKER_OUTSIDE_HINT}
+    [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
+
+[deep_sky_blue4]Commands:[/deep_sky_blue4]
+    {COMMANDS_HELP_TEXT}
+
+[deep_sky_blue4]Documentation:[/deep_sky_blue4]
+    [link=https://github.com/ArchiveBox/ArchiveBox/wiki]https://github.com/ArchiveBox/ArchiveBox/wiki[/link]{DOCKER_DOCS}
+    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage[/link]
+    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
+""")
+
+    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
+        pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
+        EXAMPLE_USAGE = f"""
+[light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]
+
+[violet]Hint:[/violet] [i]Common maintenance tasks:[/i]
+    [dark_green]archivebox[/dark_green] [green]init[/green]      [grey53]# make sure database is up-to-date (safe to run multiple times)[/grey53]
+    [dark_green]archivebox[/dark_green] [green]install[/green]   [grey53]# make sure plugins are up-to-date (wget, chrome, singlefile, etc.)[/grey53]
+    [dark_green]archivebox[/dark_green] [green]status[/green]    [grey53]# get a health checkup report on your collection[/grey53]
+    [dark_green]archivebox[/dark_green] [green]update[/green]    [grey53]# retry any previously failed or interrupted archiving tasks[/grey53]
+
+[violet]Hint:[/violet] [i]More example usage:[/i]
+    [dark_green]archivebox[/dark_green] [green]add[/green] --depth=1 "https://example.com/some/page"
+    [dark_green]archivebox[/dark_green] [green]list[/green] --sort=timestamp --csv=timestamp,downloaded_at,url,title
+    [dark_green]archivebox[/dark_green] [green]schedule[/green] --every=day --depth=1 "https://example.com/some/feed.rss"
+    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:8000[/blue]                [grey53]# Start the Web UI / API server[/grey53]
+"""
+        print(
+            Panel(
+                EXAMPLE_USAGE,
+                expand=False,
+                border_style="grey53",
+                title="[green3]:white_check_mark: A collection [light_slate_blue]DATA DIR[/light_slate_blue] is currently active[/green3]",
+                subtitle="Commands run inside this dir will only apply to this collection.",
+            ),
+        )
+    else:
+        DATA_SETUP_HELP = "\n"
+        if IN_DOCKER:
+            DATA_SETUP_HELP += "[violet]Hint:[/violet] When using Docker, you need to mount a volume to use as your data dir:\n"
+            DATA_SETUP_HELP += "    docker run [violet]-v /some/path/data:/data[/violet] archivebox/archivebox ...\n\n"
+        DATA_SETUP_HELP += "To load an [dark_blue]existing[/dark_blue] collection:\n"
+        DATA_SETUP_HELP += "    1. [green]cd[/green] ~/archivebox/data     [grey53]# go into existing [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n"
+        DATA_SETUP_HELP += f"    2. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# migrate to latest version (safe to run multiple times)[/grey53]\n"
+        DATA_SETUP_HELP += f"    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-update all plugins (wget, chrome, singlefile, etc.)[/grey53]\n"
+        DATA_SETUP_HELP += f"    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ...get help with next steps... [/grey53]\n\n"
+        DATA_SETUP_HELP += "To start a [sea_green1]new[/sea_green1] collection:\n"
+        DATA_SETUP_HELP += "    1. [green]mkdir[/green] ~/archivebox/data  [grey53]# create a new, empty [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n"
+        DATA_SETUP_HELP += "    2. [green]cd[/green] ~/archivebox/data     [grey53]# cd into the new directory[/grey53]\n"
+        DATA_SETUP_HELP += f"    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# initialize ArchiveBox in the new data dir[/grey53]\n"
+        DATA_SETUP_HELP += f"    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-install all plugins (wget, chrome, singlefile, etc.)[/grey53]\n"
+        DATA_SETUP_HELP += f"    5. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ... get help with next steps... [/grey53]\n"
+        print(
+            Panel(
+                DATA_SETUP_HELP,
+                expand=False,
+                border_style="grey53",
+                title="[red]:cross_mark: No collection is currently active[/red]",
+                subtitle="All archivebox [green]commands[/green] should be run from inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
+            ),
+        )


-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+@click.command()
+@click.option("--help", "-h", is_flag=True, help="Show help")
+def main(**kwargs):
+    """Print the ArchiveBox help message and usage"""
+    return help()
+
+
+if __name__ == "__main__":
+    main()
@@ -1,57 +1,233 @@
 #!/usr/bin/env python3

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox init'
+__package__ = "archivebox.cli"

+import os
 import sys
-import argparse
+from pathlib import Path
+from collections.abc import Mapping

-from typing import Optional, List, IO
+from rich import print
+import rich_click as click

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
-from ..main import init
+from archivebox.misc.util import docstring, enforce_types


+def _normalize_snapshot_record(link_dict: Mapping[str, object]) -> tuple[str, dict[str, object]] | None:
+    url = link_dict.get("url")
+    if not isinstance(url, str) or not url:
+        return None
+
+    record: dict[str, object] = {"url": url}
+    for key in ("timestamp", "title", "tags", "sources"):
+        value = link_dict.get(key)
+        if value is not None:
+            record[key] = value
+    return url, record
+
+
+@enforce_types
+def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
+    """Initialize a new ArchiveBox collection in the current directory"""
+
+    from archivebox.config import CONSTANTS, VERSION, DATA_DIR
+    from archivebox.config.common import SERVER_CONFIG
+    from archivebox.config.collection import write_config_file
+    from archivebox.misc.legacy import parse_json_main_index, parse_json_links_details
+    from archivebox.misc.db import apply_migrations
+
+    # if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK):
+    #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
+    #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
+
+    is_empty = not len(set(os.listdir(DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
+    existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
+    if is_empty and not existing_index:
+        print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
+        print("[green]----------------------------------------------------------------------[/green]")
+    elif existing_index:
+        # TODO: properly detect and print the existing version in current index as well
+        print(f"[green][*] Verifying and updating existing ArchiveBox collection to v{VERSION}...[/green]")
+        print("[green]----------------------------------------------------------------------[/green]")
+    else:
+        if force:
+            print("[red][!] This folder appears to already have files in it, but no index.sqlite3 is present.[/red]")
+            print("[red]    Because --force was passed, ArchiveBox will initialize anyway (which may overwrite existing files).[/red]")
+        else:
+            print(
+                "[red][X] This folder appears to already have files in it, but no index.sqlite3 present.[/red]\n\n"
+                "    You must run init in a completely empty directory, or an existing data folder.\n\n"
+                "    [violet]Hint:[/violet] To import an existing data folder make sure to cd into the folder first, \n"
+                "    then run and run 'archivebox init' to pick up where you left off.\n\n"
+                "    (Always make sure your data folder is backed up first before updating ArchiveBox)",
+            )
+            raise SystemExit(2)
+
+    if existing_index:
+        print("\n[green][*] Verifying archive folder structure...[/green]")
+    else:
+        print("\n[green][+] Building archive folder structure...[/green]")
+
+    print(
+        f"    + ./{CONSTANTS.ARCHIVE_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.SOURCES_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.LOGS_DIR.relative_to(DATA_DIR)}...",
+    )
+    Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
+    Path(CONSTANTS.ARCHIVE_DIR).mkdir(exist_ok=True)
+    Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
+
+    print(f"    + ./{CONSTANTS.CONFIG_FILE.relative_to(DATA_DIR)}...")
+
+    # create the .archivebox_id file with a unique ID for this collection
+    from archivebox.config.paths import _get_collection_id
+
+    _get_collection_id(DATA_DIR, force_create=True)
+
+    # create the ArchiveBox.conf file
+    write_config_file({"SECRET_KEY": SERVER_CONFIG.SECRET_KEY})
+
+    if os.access(CONSTANTS.DATABASE_FILE, os.F_OK):
+        print("\n[green][*] Verifying main SQL index and running any migrations needed...[/green]")
+    else:
+        print("\n[green][+] Building main SQL index and running initial migrations...[/green]")
+
+    from archivebox.config.django import setup_django
+
+    setup_django()
+
+    for migration_line in apply_migrations(DATA_DIR):
+        sys.stdout.write(f"    {migration_line}\n")
+
+    assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
+    print()
+    print(f"    √ ./{CONSTANTS.DATABASE_FILE.relative_to(DATA_DIR)}")
+
+    # from django.contrib.auth.models import User
+    # if SHELL_CONFIG.IS_TTY and not User.objects.filter(is_superuser=True).exclude(username='system').exists():
+    #     print('{green}[+] Creating admin user account...{reset}'.format(**SHELL_CONFIG.ANSI))
+    #     call_command("createsuperuser", interactive=True)
+
+    print()
+    print("[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]")
+
+    from archivebox.core.models import Snapshot
+
+    all_links = Snapshot.objects.none()
+    pending_links: dict[str, dict[str, object]] = {}
+
+    if existing_index:
+        all_links = Snapshot.objects.all()
+        print(f"    √ Loaded {all_links.count()} links from existing main index.")
+
+    if quick:
+        print("    > Skipping orphan snapshot import (quick mode)")
+    else:
+        try:
+            # Import orphaned links from legacy JSON indexes
+            orphaned_json_links: dict[str, dict[str, object]] = {}
+            for link_dict in parse_json_main_index(DATA_DIR):
+                normalized = _normalize_snapshot_record(link_dict)
+                if normalized is None:
+                    continue
+                url, record = normalized
+                if not all_links.filter(url=url).exists():
+                    orphaned_json_links[url] = record
+            if orphaned_json_links:
+                pending_links.update(orphaned_json_links)
+                print(f"    [yellow]√ Added {len(orphaned_json_links)} orphaned links from existing JSON index...[/yellow]")
+
+            orphaned_data_dir_links: dict[str, dict[str, object]] = {}
+            for link_dict in parse_json_links_details(DATA_DIR):
+                normalized = _normalize_snapshot_record(link_dict)
+                if normalized is None:
+                    continue
+                url, record = normalized
+                if not all_links.filter(url=url).exists():
+                    orphaned_data_dir_links[url] = record
+            if orphaned_data_dir_links:
+                pending_links.update(orphaned_data_dir_links)
+                print(f"    [yellow]√ Added {len(orphaned_data_dir_links)} orphaned links from existing archive directories.[/yellow]")
+
+            if pending_links:
+                for link_dict in pending_links.values():
+                    Snapshot.from_json(link_dict)
+
+            # Hint for orphaned snapshot directories
+            print()
+            print("    [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:")
+            print("        archivebox update")
+
+        except (KeyboardInterrupt, SystemExit):
+            print(file=sys.stderr)
+            print("[yellow]:stop_sign: Stopped checking archive directories due to Ctrl-C/SIGTERM[/yellow]", file=sys.stderr)
+            print("    Your archive data is safe, but you should re-run `archivebox init` to finish the process later.", file=sys.stderr)
+            print(file=sys.stderr)
+            print("    [violet]Hint:[/violet] In the future you can run a quick init without checking dirs like so:", file=sys.stderr)
+            print("        archivebox init --quick", file=sys.stderr)
+            raise SystemExit(1)
+
+    print("\n[green]----------------------------------------------------------------------[/green]")
+
+    from django.contrib.auth.models import User
+
+    if (SERVER_CONFIG.ADMIN_USERNAME and SERVER_CONFIG.ADMIN_PASSWORD) and not User.objects.filter(
+        username=SERVER_CONFIG.ADMIN_USERNAME,
+    ).exists():
+        print("[green][+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.[/green]")
+        User.objects.create_superuser(username=SERVER_CONFIG.ADMIN_USERNAME, password=SERVER_CONFIG.ADMIN_PASSWORD)
+
+    if existing_index:
+        print("[green][√] Done. Verified and updated the existing ArchiveBox collection.[/green]")
+    else:
+        print(f"[green][√] Done. A new ArchiveBox collection was initialized ({len(all_links) + len(pending_links)} links).[/green]")
+
+    CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.DEFAULT_TMP_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.DEFAULT_LIB_DIR.mkdir(parents=True, exist_ok=True)
+    (CONSTANTS.DEFAULT_LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
+
+    from archivebox.config.common import STORAGE_CONFIG
+    from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir
+
+    STORAGE_CONFIG.TMP_DIR.mkdir(parents=True, exist_ok=True)
+    STORAGE_CONFIG.LIB_DIR.mkdir(parents=True, exist_ok=True)
+    (STORAGE_CONFIG.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
+
+    working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
+    if working_tmp_dir:
+        working_tmp_dir.mkdir(parents=True, exist_ok=True)
+
+    working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
+    if working_lib_dir:
+        working_lib_dir.mkdir(parents=True, exist_ok=True)
+        (working_lib_dir / "bin").mkdir(parents=True, exist_ok=True)
+
+    if install:
+        from archivebox.cli.archivebox_install import install as install_method
+
+        install_method()
+
+    if Snapshot.objects.count() < 25:  # hide the hints for experienced users
+        print()
+        print("    [violet]Hint:[/violet] To view your archive index, run:")
+        print(
+            "        archivebox server  # then visit [deep_sky_blue4][link=http://127.0.0.1:8000]http://127.0.0.1:8000[/link][/deep_sky_blue4]",
+        )
+        print()
+        print("    To add new links, you can run:")
+        print("        archivebox add < ~/some/path/to/list_of_links.txt")
+        print()
+        print("    For more usage and examples, run:")
+        print("        archivebox help")
+
+
+@click.command()
+@click.option("--force", "-f", is_flag=True, help="Ignore unrecognized files in current directory and initialize anyway")
+@click.option("--quick", "-q", is_flag=True, help="Run any updates or migrations without rechecking all snapshot dirs")
+@click.option("--install", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
@docstring(init.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=init.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        '--force', # '-f',
-        action='store_true',
-        help='Ignore unrecognized files in current directory and initialize anyway',
-    )
-    parser.add_argument(
-        '--quick', '-q',
-        action='store_true',
-        help='Run any updates or migrations without rechecking all snapshot dirs',
-    )
-    parser.add_argument(
-        '--install', #'-s',
-        action='store_true',
-        help='Automatically install dependencies and extras used for archiving',
-    )
-    parser.add_argument(
-        '--setup', #'-s',
-        action='store_true',
-        help='DEPRECATED: equivalent to --install',
-    )
-    command = parser.parse_args(args or ())
-    reject_stdin(__command__, stdin)
+def main(**kwargs) -> None:
+    init(**kwargs)

-    init(
-        force=command.force,
-        quick=command.quick,
-        install=command.install or command.setup,
-        out_dir=pwd or DATA_DIR,
-    )
-    

-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+if __name__ == "__main__":
+    main()
@@ -1,56 +1,104 @@
 #!/usr/bin/env python3

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox install'
+__package__ = "archivebox.cli"

-import sys
-import argparse
-from pathlib import Path
-from typing import Optional, List, IO
+import os

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
-from ..main import install
+import rich_click as click
+from rich import print
+
+from archivebox.misc.util import docstring, enforce_types


+@enforce_types
+def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bool = False) -> None:
+    """Detect and install ArchiveBox dependencies by running the abx-dl install flow
+
+    Examples:
+        archivebox install                              # Install all dependencies
+        archivebox install wget curl                    # Install only wget and curl
+        archivebox install --binproviders=pip yt-dlp    # Install yt-dlp using only pip
+        archivebox install --binproviders=brew,apt      # Install all deps using only brew or apt
+    """
+
+    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
+    from archivebox.config.paths import ARCHIVE_DIR
+    from archivebox.misc.logging import stderr
+    from archivebox.cli.archivebox_init import init
+
+    if not (os.access(ARCHIVE_DIR, os.R_OK) and ARCHIVE_DIR.is_dir()):
+        init()  # must init full index because we need a db to store Binary entries in
+
+    # Show what we're installing
+    if binaries:
+        print(f"\n[green][+] Installing specific binaries: {', '.join(binaries)}[/green]")
+    else:
+        print("\n[green][+] Detecting and installing all ArchiveBox dependencies...[/green]")
+
+    if binproviders != "*":
+        print(f"[green][+] Using providers: {binproviders}[/green]")
+
+    if IS_ROOT:
+        EUID = os.geteuid()
+        print()
+        print(f"[yellow]:warning:  Running as UID=[blue]{EUID}[/blue].[/yellow]")
+        print(f"    DATA_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].")
+        print()
+
+    if dry_run:
+        print("[dim]Dry run - would run the abx-dl install flow[/dim]")
+        return
+
+    # Set up Django
+    from archivebox.config.django import setup_django
+
+    setup_django()
+
+    plugin_names = list(binaries)
+    if binproviders != "*":
+        plugin_names.extend(provider.strip() for provider in binproviders.split(",") if provider.strip())
+
+    print("[+] Running installer via abx-dl bus...")
+    print()
+
+    from archivebox.services.runner import run_install
+
+    run_install(plugin_names=plugin_names or None)
+
+    print()
+
+    # Check for superuser
+    from django.contrib.auth import get_user_model
+
+    User = get_user_model()
+
+    if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
+        stderr("\n[+] Don't forget to create a new admin user for the Web UI...", color="green")
+        stderr("    archivebox manage createsuperuser")
+
+    print()
+
+    # Show version to display full status including installed binaries
+    # Django is already loaded, so just import and call the function directly
+    from archivebox.cli.archivebox_version import version as show_version
+
+    show_version(quiet=False)
+
+
+@click.command()
+@click.argument("binaries", nargs=-1, type=str, required=False)
+@click.option(
+    "--binproviders",
+    "-p",
+    default="*",
+    help="Comma-separated list of providers to use (pip,npm,brew,apt,env,custom) or * for all",
+    show_default=True,
+)
+@click.option("--dry-run", "-d", is_flag=True, help="Show what would happen without actually running", default=False)
@docstring(install.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=install.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        '--binproviders', '-p',
-        type=str,
-        help='Select binproviders to use DEFAULT=env,apt,brew,sys_pip,venv_pip,lib_pip,pipx,sys_npm,lib_npm,puppeteer,playwright (all)',
-        default=None,
-    )
-    parser.add_argument(
-        '--binaries', '-b',
-        type=str,
-        help='Select binaries to install DEFAULT=curl,wget,git,yt-dlp,chrome,single-file,readability-extractor,postlight-parser,... (all)',
-        default=None,
-    )
-    parser.add_argument(
-        '--dry-run', '-d',
-        action='store_true',
-        help='Show what would be installed without actually installing anything',
-        default=False,
-    )
-    command = parser.parse_args(args or ())   # noqa
-    reject_stdin(__command__, stdin)
+def main(**kwargs) -> None:
+    install(**kwargs)

-    install(
-        # force=command.force,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
-        binaries=command.binaries.split(',') if command.binaries else None,
-        binproviders=command.binproviders.split(',') if command.binproviders else None,
-        dry_run=command.dry_run,
-    )
-    

-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+if __name__ == "__main__":
+    main()
@@ -1,139 +1,57 @@
 #!/usr/bin/env python3

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox list'
+__package__ = "archivebox.cli"
+__command__ = "archivebox list"

 import sys
-import argparse
-from pathlib import Path
-from typing import Optional, List, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..main import list_all
-from ..index import (
-    LINK_FILTERS,
-    get_indexed_folders,
-    get_archived_folders,
-    get_unarchived_folders,
-    get_present_folders,
-    get_valid_folders,
-    get_invalid_folders,
-    get_duplicate_folders,
-    get_orphaned_folders,
-    get_corrupted_folders,
-    get_unrecognized_folders,
-)
-from ..logging_util import SmartFormatter, reject_stdin, stderr
+import rich_click as click
+
+from archivebox.cli.archivebox_snapshot import list_snapshots


-@docstring(list_all.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=list_all.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
+@click.command()
+@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
+@click.option("--url__icontains", help="Filter by URL contains")
+@click.option("--url__istartswith", help="Filter by URL starts with")
+@click.option("--tag", "-t", help="Filter by tag name")
+@click.option("--crawl-id", help="Filter by crawl ID")
+@click.option("--limit", "-n", type=int, help="Limit number of results")
+@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
+@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
+@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
+@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
+@click.argument("query", nargs=-1)
+def main(
+    status: str | None,
+    url__icontains: str | None,
+    url__istartswith: str | None,
+    tag: str | None,
+    crawl_id: str | None,
+    limit: int | None,
+    sort: str | None,
+    csv: str | None,
+    with_headers: bool,
+    search: str | None,
+    query: tuple[str, ...],
+) -> None:
+    """List Snapshots."""
+    sys.exit(
+        list_snapshots(
+            status=status,
+            url__icontains=url__icontains,
+            url__istartswith=url__istartswith,
+            tag=tag,
+            crawl_id=crawl_id,
+            limit=limit,
+            sort=sort,
+            csv=csv,
+            with_headers=with_headers,
+            search=search,
+            query=" ".join(query),
+        ),
    )
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-        '--csv', #'-c',
-        type=str,
-        help="Print the output in CSV format with the given columns, e.g.: timestamp,url,extension",
-        default=None,
-    )
-    group.add_argument(
-        '--json', #'-j',
-        action='store_true',
-        help="Print the output in JSON format with all columns included",
-    )
-    group.add_argument(
-        '--html',
-        action='store_true',
-        help="Print the output in HTML format"
-    )
-    parser.add_argument(
-        '--with-headers',
-        action='store_true',
-        help='Include the headers in the output document' 
-    )
-    parser.add_argument(
-        '--sort', #'-s',
-        type=str,
-        help="List the links sorted using the given key, e.g. timestamp or updated",
-        default=None,
-    )
-    parser.add_argument(
-        '--before', #'-b',
-        type=float,
-        help="List only links bookmarked before (less than) the given timestamp",
-        default=None,
-    )
-    parser.add_argument(
-        '--after', #'-a',
-        type=float,
-        help="List only links bookmarked after (greater than or equal to) the given timestamp",
-        default=None,
-    )
-    parser.add_argument(
-        '--status',
-        type=str,
-        choices=('indexed', 'archived', 'unarchived', 'present', 'valid', 'invalid', 'duplicate', 'orphaned', 'corrupted', 'unrecognized'),
-        default='indexed',
-        help=(
-            'List only links or data directories that have the given status\n'
-            f'    indexed       {get_indexed_folders.__doc__} (the default)\n'
-            f'    archived      {get_archived_folders.__doc__}\n'
-            f'    unarchived    {get_unarchived_folders.__doc__}\n'
-            '\n'
-            f'    present       {get_present_folders.__doc__}\n'
-            f'    valid         {get_valid_folders.__doc__}\n'
-            f'    invalid       {get_invalid_folders.__doc__}\n'
-            '\n'
-            f'    duplicate     {get_duplicate_folders.__doc__}\n'
-            f'    orphaned      {get_orphaned_folders.__doc__}\n'
-            f'    corrupted     {get_corrupted_folders.__doc__}\n'
-            f'    unrecognized  {get_unrecognized_folders.__doc__}\n'
-        )
-    )
-    parser.add_argument(
-        '--filter-type', '-t',
-        type=str,
-        choices=(*LINK_FILTERS.keys(), 'search'),
-        default='exact',
-        help='Type of pattern matching to use when filtering URLs',
-    )
-    parser.add_argument(
-        'filter_patterns',
-        nargs='*',
-        type=str,
-        default=None,
-        help='List only URLs matching these filter patterns'
-    )
-    command = parser.parse_args(args or ())
-    reject_stdin(stdin)

-    if command.with_headers and not (command.json or command.html or command.csv):
-        stderr(
-            '[X] --with-headers can only be used with --json, --html or --csv options\n',
-            color='red',
-        )
-        raise SystemExit(2)

-    matching_folders = list_all(
-        filter_patterns=command.filter_patterns,
-        filter_type=command.filter_type,
-        status=command.status,
-        after=command.after,
-        before=command.before,
-        sort=command.sort,
-        csv=command.csv,
-        json=command.json,
-        html=command.html,
-        with_headers=command.with_headers,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
-    )
-    raise SystemExit(not matching_folders)
-
-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+"""
+archivebox machine <action> [--filters]
+
+Manage Machine records (system-managed, mostly read-only).
+
+Machine records track the host machines where ArchiveBox runs.
+They are created automatically by the system and are primarily for debugging.
+
+Actions:
+    list    - List Machines as JSONL (with optional filters)
+
+Examples:
+    # List all machines
+    archivebox machine list
+
+    # List machines by hostname
+    archivebox machine list --hostname__icontains=myserver
+"""
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox machine"
+
+import sys
+
+import rich_click as click
+from rich import print as rprint
+
+from archivebox.cli.cli_utils import apply_filters
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+
+def list_machines(
+    hostname__icontains: str | None = None,
+    os_platform: str | None = None,
+    limit: int | None = None,
+) -> int:
+    """
+    List Machines as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.machine.models import Machine
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Machine.objects.all().order_by("-created_at")
+
+    # Apply filters
+    filter_kwargs = {
+        "hostname__icontains": hostname__icontains,
+        "os_platform": os_platform,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for machine in queryset:
+        if is_tty:
+            rprint(f"[cyan]{machine.hostname:30}[/cyan] [dim]{machine.os_platform:10}[/dim] {machine.id}")
+        else:
+            write_record(machine.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} machines[/dim]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+
+@click.group()
+def main():
+    """Manage Machine records (read-only, system-managed)."""
+    pass
+
+
+@main.command("list")
+@click.option("--hostname__icontains", help="Filter by hostname contains")
+@click.option("--os-platform", help="Filter by OS platform")
+@click.option("--limit", "-n", type=int, help="Limit number of results")
+def list_cmd(hostname__icontains: str | None, os_platform: str | None, limit: int | None):
+    """List Machines as JSONL."""
+    sys.exit(
+        list_machines(
+            hostname__icontains=hostname__icontains,
+            os_platform=os_platform,
+            limit=limit,
+        ),
+    )
+
+
+if __name__ == "__main__":
+    main()
@@ -1,24 +1,34 @@
 #!/usr/bin/env python3

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox manage'
+__package__ = "archivebox.cli"

-import sys
-from pathlib import Path
-from typing import Optional, List, IO
-
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..main import manage
+import rich_click as click
+from archivebox.misc.util import docstring, enforce_types


+@enforce_types
+def manage(args: list[str] | None = None) -> None:
+    """Run an ArchiveBox Django management command"""
+
+    from archivebox.config.common import SHELL_CONFIG
+    from archivebox.misc.logging import stderr
+
+    if (args and "createsuperuser" in args) and (SHELL_CONFIG.IN_DOCKER and not SHELL_CONFIG.IS_TTY):
+        stderr("[!] Warning: you need to pass -it to use interactive commands in docker", color="lightyellow")
+        stderr("    docker run -it archivebox manage {}".format(" ".join(args or ["..."])), color="lightyellow")
+        stderr("")
+
+    from django.core.management import execute_from_command_line
+
+    execute_from_command_line(["manage.py", *(args or ["help"])])
+
+
+@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
+@click.argument("args", nargs=-1)
@docstring(manage.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    manage(
-        args=args,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
-    )
+def main(args: list[str] | None = None) -> None:
+    manage(args=args)


-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""
+archivebox mcp
+
+Start the Model Context Protocol (MCP) server in stdio mode.
+Exposes all ArchiveBox CLI commands as MCP tools for AI agents.
+"""
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox mcp"
+
+import rich_click as click
+
+from archivebox.misc.util import docstring, enforce_types
+
+
+@enforce_types
+def mcp():
+    """
+    Start the MCP server in stdio mode for AI agent control.
+
+    The MCP (Model Context Protocol) server exposes all ArchiveBox CLI commands
+    as tools that AI agents can discover and execute. It communicates via JSON-RPC
+    2.0 over stdin/stdout.
+
+    Example usage with an MCP client:
+        archivebox mcp < requests.jsonl > responses.jsonl
+
+    Or interactively:
+        archivebox mcp
+        {"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}
+        {"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}
+    """
+
+    from archivebox.mcp.server import run_mcp_server
+
+    # Run the stdio server (blocks until stdin closes)
+    run_mcp_server()
+
+
+@click.command()
+@docstring(mcp.__doc__)
+def main(**kwargs):
+    """Start the MCP server in stdio mode"""
+    mcp()
+
+
+if __name__ == "__main__":
+    main()
@@ -1,73 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = 'archivebox.cli'
-__command__ = 'archivebox oneshot'
-
-import sys
-import argparse
-
-from pathlib import Path
-from typing import List, Optional, IO
-
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, accept_stdin, stderr
-from ..main import oneshot
-
-
-@docstring(oneshot.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=oneshot.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        'url',
-        type=str,
-        default=None,
-        help=(
-            'URLs or paths to archive e.g.:\n'
-            '    https://getpocket.com/users/USERNAME/feed/all\n'
-            '    https://example.com/some/rss/feed.xml\n'
-            '    https://example.com\n'
-            '    ~/Downloads/firefox_bookmarks_export.html\n'
-            '    ~/Desktop/sites_list.csv\n'
-        )
-    )
-    parser.add_argument(
-        "--extract",
-        type=str,
-        help="Pass a list of the extractors to be used. If the method name is not correct, it will be ignored. \
-              This does not take precedence over the configuration",
-        default=""
-    )
-    parser.add_argument(
-        '--out-dir',
-        type=str,
-        default=DATA_DIR,
-        help= "Path to save the single archive folder to, e.g. ./example.com_archive"
-    )
-    command = parser.parse_args(args or ())
-    stdin_url = None
-    url = command.url
-    if not url:
-        stdin_url = accept_stdin(stdin)
-
-    if (stdin_url and url) or (not stdin and not url):
-        stderr(
-            '[X] You must pass a URL/path to add via stdin or CLI arguments.\n',
-            color='red',
-        )
-        raise SystemExit(2)
-    
-    oneshot(
-        url=stdin_url or url,
-        out_dir=Path(command.out_dir).resolve(),
-        extractors=command.extract,
-    )
-
-
-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -0,0 +1,776 @@
+#!/usr/bin/env python3
+
+"""
+archivebox persona <action> [args...] [--filters]
+
+Manage Persona records (browser profiles for archiving).
+
+Actions:
+    create  - Create Personas
+    list    - List Personas as JSONL (with optional filters)
+    update  - Update Personas from stdin JSONL
+    delete  - Delete Personas from stdin JSONL
+
+Examples:
+    # Create a new persona
+    archivebox persona create work
+    archivebox persona create --import=chrome personal
+    archivebox persona create --import=edge work
+
+    # List all personas
+    archivebox persona list
+
+    # Delete a persona
+    archivebox persona list --name=old | archivebox persona delete --yes
+"""
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox persona"
+
+import os
+import sys
+import shutil
+import platform
+import subprocess
+import tempfile
+import json
+from pathlib import Path
+from collections.abc import Iterable
+from collections import OrderedDict
+
+import rich_click as click
+from rich import print as rprint
+
+from archivebox.cli.cli_utils import apply_filters
+from archivebox.personas import importers as persona_importers
+
+
+# =============================================================================
+# Browser Profile Locations
+# =============================================================================
+
+
+def get_chrome_user_data_dir() -> Path | None:
+    """Get the default Chrome user data directory for the current platform."""
+    system = platform.system()
+    home = Path.home()
+
+    if system == "Darwin":  # macOS
+        candidates = [
+            home / "Library" / "Application Support" / "Google" / "Chrome",
+            home / "Library" / "Application Support" / "Chromium",
+        ]
+    elif system == "Linux":
+        candidates = [
+            home / ".config" / "google-chrome",
+            home / ".config" / "chromium",
+            home / ".config" / "chrome",
+            home / "snap" / "chromium" / "common" / "chromium",
+        ]
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = [
+            local_app_data / "Google" / "Chrome" / "User Data",
+            local_app_data / "Chromium" / "User Data",
+        ]
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate.exists() and (candidate / "Default").exists():
+            return candidate
+
+    return None
+
+
+def get_brave_user_data_dir() -> Path | None:
+    """Get the default Brave user data directory for the current platform."""
+    system = platform.system()
+    home = Path.home()
+
+    if system == "Darwin":
+        candidates = [
+            home / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser",
+        ]
+    elif system == "Linux":
+        candidates = [
+            home / ".config" / "BraveSoftware" / "Brave-Browser",
+        ]
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = [
+            local_app_data / "BraveSoftware" / "Brave-Browser" / "User Data",
+        ]
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate.exists() and (candidate / "Default").exists():
+            return candidate
+
+    return None
+
+
+def get_edge_user_data_dir() -> Path | None:
+    """Get the default Edge user data directory for the current platform."""
+    system = platform.system()
+    home = Path.home()
+
+    if system == "Darwin":
+        candidates = [
+            home / "Library" / "Application Support" / "Microsoft Edge",
+        ]
+    elif system == "Linux":
+        candidates = [
+            home / ".config" / "microsoft-edge",
+            home / ".config" / "microsoft-edge-beta",
+            home / ".config" / "microsoft-edge-dev",
+        ]
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = [
+            local_app_data / "Microsoft" / "Edge" / "User Data",
+        ]
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate.exists() and (candidate / "Default").exists():
+            return candidate
+
+    return None
+
+
+def get_browser_binary(browser: str) -> str | None:
+    system = platform.system()
+    home = Path.home()
+    browser = browser.lower()
+
+    if system == "Darwin":
+        candidates = {
+            "chrome": ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"],
+            "chromium": ["/Applications/Chromium.app/Contents/MacOS/Chromium"],
+            "brave": ["/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"],
+            "edge": ["/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"],
+        }.get(browser, [])
+    elif system == "Linux":
+        candidates = {
+            "chrome": [
+                "/usr/bin/google-chrome",
+                "/usr/bin/google-chrome-stable",
+                "/usr/bin/google-chrome-beta",
+                "/usr/bin/google-chrome-unstable",
+            ],
+            "chromium": ["/usr/bin/chromium", "/usr/bin/chromium-browser"],
+            "brave": ["/usr/bin/brave-browser", "/usr/bin/brave-browser-beta", "/usr/bin/brave-browser-nightly"],
+            "edge": [
+                "/usr/bin/microsoft-edge",
+                "/usr/bin/microsoft-edge-stable",
+                "/usr/bin/microsoft-edge-beta",
+                "/usr/bin/microsoft-edge-dev",
+            ],
+        }.get(browser, [])
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = {
+            "chrome": [
+                str(local_app_data / "Google" / "Chrome" / "Application" / "chrome.exe"),
+                "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
+                "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
+            ],
+            "chromium": [str(local_app_data / "Chromium" / "Application" / "chrome.exe")],
+            "brave": [
+                str(local_app_data / "BraveSoftware" / "Brave-Browser" / "Application" / "brave.exe"),
+                "C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
+                "C:\\Program Files (x86)\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
+            ],
+            "edge": [
+                str(local_app_data / "Microsoft" / "Edge" / "Application" / "msedge.exe"),
+                "C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe",
+                "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
+            ],
+        }.get(browser, [])
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate and Path(candidate).exists():
+            return candidate
+
+    return None
+
+
+BROWSER_PROFILE_FINDERS = {
+    "chrome": get_chrome_user_data_dir,
+    "chromium": get_chrome_user_data_dir,  # Same locations
+    "brave": get_brave_user_data_dir,
+    "edge": get_edge_user_data_dir,
+}
+
+CHROMIUM_BROWSERS = {"chrome", "chromium", "brave", "edge"}
+
+
+# =============================================================================
+# Cookie Extraction via CDP
+# =============================================================================
+
+NETSCAPE_COOKIE_HEADER = [
+    "# Netscape HTTP Cookie File",
+    "# https://curl.se/docs/http-cookies.html",
+    "# This file was generated by ArchiveBox persona cookie extraction",
+    "#",
+    "# Format: domain\\tincludeSubdomains\\tpath\\tsecure\\texpiry\\tname\\tvalue",
+    "",
+]
+
+
+def _parse_netscape_cookies(path: Path) -> "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]":
+    cookies = OrderedDict()
+    if not path.exists():
+        return cookies
+
+    for line in path.read_text().splitlines():
+        if not line or line.startswith("#"):
+            continue
+        parts = line.split("\t")
+        if len(parts) < 7:
+            continue
+        domain, include_subdomains, cookie_path, secure, expiry, name, value = parts[:7]
+        key = (domain, cookie_path, name)
+        cookies[key] = (domain, include_subdomains, cookie_path, secure, expiry, name, value)
+    return cookies
+
+
+def _write_netscape_cookies(path: Path, cookies: "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]") -> None:
+    lines = list(NETSCAPE_COOKIE_HEADER)
+    for cookie in cookies.values():
+        lines.append("\t".join(cookie))
+    path.write_text("\n".join(lines) + "\n")
+
+
+def _merge_netscape_cookies(existing_file: Path, new_file: Path) -> None:
+    existing = _parse_netscape_cookies(existing_file)
+    new = _parse_netscape_cookies(new_file)
+    for key, cookie in new.items():
+        existing[key] = cookie
+    _write_netscape_cookies(existing_file, existing)
+
+
+def extract_cookies_via_cdp(
+    user_data_dir: Path,
+    output_file: Path,
+    profile_dir: str | None = None,
+    chrome_binary: str | None = None,
+) -> bool:
+    """
+    Launch Chrome with the given user data dir and extract cookies via CDP.
+
+    Returns True if successful, False otherwise.
+    """
+    from archivebox.config.common import STORAGE_CONFIG
+
+    # Find the cookie extraction script
+    chrome_plugin_dir = Path(__file__).parent.parent / "plugins" / "chrome"
+    extract_script = chrome_plugin_dir / "extract_cookies.js"
+
+    if not extract_script.exists():
+        rprint(f"[yellow]Cookie extraction script not found at {extract_script}[/yellow]", file=sys.stderr)
+        return False
+
+    # Get node modules dir
+    node_modules_dir = STORAGE_CONFIG.LIB_DIR / "npm" / "node_modules"
+
+    # Set up environment
+    env = os.environ.copy()
+    env["NODE_MODULES_DIR"] = str(node_modules_dir)
+    env["CHROME_USER_DATA_DIR"] = str(user_data_dir)
+    env["CHROME_HEADLESS"] = "true"
+    if chrome_binary:
+        env["CHROME_BINARY"] = str(chrome_binary)
+    output_path = output_file
+    temp_output = None
+    temp_dir = None
+    if output_file.exists():
+        temp_dir = Path(tempfile.mkdtemp(prefix="ab_cookies_"))
+        temp_output = temp_dir / "cookies.txt"
+        output_path = temp_output
+    if profile_dir:
+        extra_arg = f"--profile-directory={profile_dir}"
+        existing_extra = env.get("CHROME_ARGS_EXTRA", "").strip()
+        args_list = []
+        if existing_extra:
+            if existing_extra.startswith("["):
+                try:
+                    parsed = json.loads(existing_extra)
+                    if isinstance(parsed, list):
+                        args_list.extend(str(x) for x in parsed)
+                except Exception:
+                    args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()])
+            else:
+                args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()])
+        args_list.append(extra_arg)
+        env["CHROME_ARGS_EXTRA"] = json.dumps(args_list)
+
+    env["COOKIES_OUTPUT_FILE"] = str(output_path)
+
+    try:
+        result = subprocess.run(
+            ["node", str(extract_script)],
+            env=env,
+            capture_output=True,
+            text=True,
+            timeout=60,
+        )
+
+        if result.returncode == 0:
+            if temp_output and temp_output.exists():
+                _merge_netscape_cookies(output_file, temp_output)
+            return True
+        else:
+            rprint(f"[yellow]Cookie extraction failed: {result.stderr}[/yellow]", file=sys.stderr)
+            return False
+
+    except subprocess.TimeoutExpired:
+        rprint("[yellow]Cookie extraction timed out[/yellow]", file=sys.stderr)
+        return False
+    except FileNotFoundError:
+        rprint("[yellow]Node.js not found. Cannot extract cookies.[/yellow]", file=sys.stderr)
+        return False
+    except Exception as e:
+        rprint(f"[yellow]Cookie extraction error: {e}[/yellow]", file=sys.stderr)
+        return False
+    finally:
+        if temp_dir and temp_dir.exists():
+            shutil.rmtree(temp_dir, ignore_errors=True)
+
+
+# =============================================================================
+# Validation Helpers
+# =============================================================================
+
+
+def validate_persona_name(name: str) -> tuple[bool, str]:
+    """
+    Validate persona name to prevent path traversal attacks.
+
+    Returns:
+        (is_valid, error_message): tuple indicating if name is valid
+    """
+    if not name or not name.strip():
+        return False, "Persona name cannot be empty"
+
+    # Check for path separators
+    if "/" in name or "\\" in name:
+        return False, "Persona name cannot contain path separators (/ or \\)"
+
+    # Check for parent directory references
+    if ".." in name:
+        return False, "Persona name cannot contain parent directory references (..)"
+
+    # Check for hidden files/directories
+    if name.startswith("."):
+        return False, "Persona name cannot start with a dot (.)"
+
+    # Ensure name doesn't contain null bytes or other dangerous chars
+    if "\x00" in name or "\n" in name or "\r" in name:
+        return False, "Persona name contains invalid characters"
+
+    return True, ""
+
+
+def ensure_path_within_personas_dir(persona_path: Path) -> bool:
+    """
+    Verify that a persona path is within PERSONAS_DIR.
+
+    This is a safety check to prevent path traversal attacks where
+    a malicious persona name could cause operations on paths outside
+    the expected PERSONAS_DIR.
+
+    Returns:
+        True if path is safe, False otherwise
+    """
+    from archivebox.config.constants import CONSTANTS
+
+    try:
+        # Resolve both paths to absolute paths
+        personas_dir = CONSTANTS.PERSONAS_DIR.resolve()
+        resolved_path = persona_path.resolve()
+
+        # Check if resolved_path is a child of personas_dir
+        return resolved_path.is_relative_to(personas_dir)
+    except (ValueError, RuntimeError):
+        return False
+
+
+# =============================================================================
+# CREATE
+# =============================================================================
+
+
+def create_personas(
+    names: Iterable[str],
+    import_from: str | None = None,
+    profile: str | None = None,
+) -> int:
+    """
+    Create Personas from names.
+
+    If --import is specified, copy the browser profile to the persona directory
+    and extract cookies.
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.personas.models import Persona
+
+    is_tty = sys.stdout.isatty()
+    name_list = list(names) if names else []
+
+    if not name_list:
+        rprint("[yellow]No persona names provided. Pass names as arguments.[/yellow]", file=sys.stderr)
+        return 1
+
+    # Validate import source if specified
+    source_profile_dir = None
+    if import_from:
+        import_from = import_from.lower()
+        if import_from not in BROWSER_PROFILE_FINDERS:
+            rprint(f"[red]Unknown browser: {import_from}[/red]", file=sys.stderr)
+            rprint(f"[dim]Supported browsers: {', '.join(BROWSER_PROFILE_FINDERS.keys())}[/dim]", file=sys.stderr)
+            return 1
+
+        source_profile_dir = BROWSER_PROFILE_FINDERS[import_from]()
+        if not source_profile_dir:
+            rprint(f"[red]Could not find {import_from} profile directory[/red]", file=sys.stderr)
+            return 1
+
+        rprint(f"[dim]Found {import_from} profile: {source_profile_dir}[/dim]", file=sys.stderr)
+
+        if profile is None and (source_profile_dir / "Default").exists():
+            profile = "Default"
+
+        browser_binary = get_browser_binary(import_from)
+        if browser_binary:
+            rprint(f"[dim]Using {import_from} binary: {browser_binary}[/dim]", file=sys.stderr)
+
+    created_count = 0
+    for name in name_list:
+        name = name.strip()
+        if not name:
+            continue
+
+        # Validate persona name to prevent path traversal
+        is_valid, error_msg = persona_importers.validate_persona_name(name)
+        if not is_valid:
+            rprint(f'[red]Invalid persona name "{name}": {error_msg}[/red]', file=sys.stderr)
+            continue
+
+        persona, created = Persona.objects.get_or_create(name=name)
+
+        if created:
+            persona.ensure_dirs()
+            created_count += 1
+            rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr)
+        else:
+            rprint(f"[dim]Persona already exists: {name}[/dim]", file=sys.stderr)
+
+        cookies_file = Path(persona.path) / "cookies.txt"
+
+        # Import browser profile if requested
+        if import_from in CHROMIUM_BROWSERS and source_profile_dir is not None:
+            try:
+                import_source = persona_importers.resolve_browser_import_source(import_from, profile_dir=profile)
+                import_result = persona_importers.import_persona_from_source(
+                    persona,
+                    import_source,
+                    copy_profile=True,
+                    import_cookies=True,
+                    capture_storage=False,
+                )
+            except Exception as e:
+                rprint(f"[red]Failed to import browser profile: {e}[/red]", file=sys.stderr)
+                return 1
+
+            if import_result.profile_copied:
+                rprint("[green]Copied browser profile to persona[/green]", file=sys.stderr)
+            if import_result.cookies_imported:
+                rprint(f"[green]Extracted cookies to {cookies_file}[/green]", file=sys.stderr)
+            elif not import_result.profile_copied:
+                rprint("[yellow]Could not import cookies automatically.[/yellow]", file=sys.stderr)
+
+            for warning in import_result.warnings:
+                rprint(f"[yellow]{warning}[/yellow]", file=sys.stderr)
+
+        if not is_tty:
+            write_record(
+                {
+                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "name": persona.name,
+                    "path": str(persona.path),
+                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
+                    "COOKIES_FILE": persona.COOKIES_FILE,
+                },
+            )
+
+    rprint(f"[green]Created {created_count} new persona(s)[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+
+def list_personas(
+    name: str | None = None,
+    name__icontains: str | None = None,
+    limit: int | None = None,
+) -> int:
+    """
+    List Personas as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.personas.models import Persona
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Persona.objects.all().order_by("name")
+
+    # Apply filters
+    filter_kwargs = {
+        "name": name,
+        "name__icontains": name__icontains,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for persona in queryset:
+        cookies_status = "[green]✓[/green]" if persona.COOKIES_FILE else "[dim]✗[/dim]"
+        chrome_status = "[green]✓[/green]" if Path(persona.CHROME_USER_DATA_DIR).exists() else "[dim]✗[/dim]"
+
+        if is_tty:
+            rprint(f"[cyan]{persona.name:20}[/cyan] cookies:{cookies_status} chrome:{chrome_status} [dim]{persona.path}[/dim]")
+        else:
+            write_record(
+                {
+                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "name": persona.name,
+                    "path": str(persona.path),
+                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
+                    "COOKIES_FILE": persona.COOKIES_FILE,
+                },
+            )
+        count += 1
+
+    rprint(f"[dim]Listed {count} persona(s)[/dim]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# UPDATE
+# =============================================================================
+
+
+def update_personas(name: str | None = None) -> int:
+    """
+    Update Personas from stdin JSONL.
+
+    Reads Persona records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from archivebox.misc.jsonl import read_stdin, write_record
+    from archivebox.personas.models import Persona
+
+    is_tty = sys.stdout.isatty()
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        persona_id = record.get("id")
+        old_name = record.get("name")
+
+        if not persona_id and not old_name:
+            continue
+
+        try:
+            if persona_id:
+                persona = Persona.objects.get(id=persona_id)
+            else:
+                persona = Persona.objects.get(name=old_name)
+
+            # Apply updates from CLI flags
+            if name:
+                # Validate new name to prevent path traversal
+                is_valid, error_msg = persona_importers.validate_persona_name(name)
+                if not is_valid:
+                    rprint(f'[red]Invalid new persona name "{name}": {error_msg}[/red]', file=sys.stderr)
+                    continue
+
+                # Rename the persona directory too
+                old_path = persona.path
+                persona.name = name
+                new_path = persona.path
+
+                if old_path.exists() and old_path != new_path:
+                    shutil.move(str(old_path), str(new_path))
+
+                persona.save()
+
+            updated_count += 1
+
+            if not is_tty:
+                write_record(
+                    {
+                        "id": str(persona.id) if hasattr(persona, "id") else None,
+                        "name": persona.name,
+                        "path": str(persona.path),
+                    },
+                )
+
+        except Persona.DoesNotExist:
+            rprint(f"[yellow]Persona not found: {persona_id or old_name}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} persona(s)[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# DELETE
+# =============================================================================
+
+
+def delete_personas(yes: bool = False, dry_run: bool = False) -> int:
+    """
+    Delete Personas from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
+    from archivebox.personas.models import Persona
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    # Collect persona IDs or names
+    persona_ids = []
+    persona_names = []
+    for r in records:
+        if r.get("id"):
+            persona_ids.append(r["id"])
+        elif r.get("name"):
+            persona_names.append(r["name"])
+
+    if not persona_ids and not persona_names:
+        rprint("[yellow]No valid persona IDs or names in input[/yellow]", file=sys.stderr)
+        return 1
+
+    from django.db.models import Q
+
+    query = Q()
+    if persona_ids:
+        query |= Q(id__in=persona_ids)
+    if persona_names:
+        query |= Q(name__in=persona_names)
+
+    personas = Persona.objects.filter(query)
+    count = personas.count()
+
+    if count == 0:
+        rprint("[yellow]No matching personas found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} persona(s) (dry run)[/yellow]", file=sys.stderr)
+        for persona in personas:
+            rprint(f"  {persona.name} ({persona.path})", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Delete persona directories and database records
+    deleted_count = 0
+    for persona in personas:
+        persona_path = persona.path
+
+        # Safety check: ensure path is within PERSONAS_DIR before deletion
+        if not ensure_path_within_personas_dir(persona_path):
+            rprint(f'[red]Security error: persona path "{persona_path}" is outside PERSONAS_DIR. Skipping deletion.[/red]', file=sys.stderr)
+            continue
+
+        if persona_path.exists():
+            shutil.rmtree(persona_path)
+        persona.delete()
+        deleted_count += 1
+
+    rprint(f"[green]Deleted {deleted_count} persona(s)[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+
+@click.group()
+def main():
+    """Manage Persona records (browser profiles)."""
+    pass
+
+
+@main.command("create")
+@click.argument("names", nargs=-1)
+@click.option("--import", "import_from", help="Import profile from browser (chrome, chromium, brave, edge)")
+@click.option("--profile", help="Profile directory name under the user data dir (e.g. Default, Profile 1)")
+def create_cmd(names: tuple, import_from: str | None, profile: str | None):
+    """Create Personas, optionally importing from a browser profile."""
+    sys.exit(create_personas(names, import_from=import_from, profile=profile))
+
+
+@main.command("list")
+@click.option("--name", help="Filter by exact name")
+@click.option("--name__icontains", help="Filter by name contains")
+@click.option("--limit", "-n", type=int, help="Limit number of results")
+def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
+    """List Personas as JSONL."""
+    sys.exit(list_personas(name=name, name__icontains=name__icontains, limit=limit))
+
+
+@main.command("update")
+@click.option("--name", "-n", help="Set new name")
+def update_cmd(name: str | None):
+    """Update Personas from stdin JSONL."""
+    sys.exit(update_personas(name=name))
+
+
+@main.command("delete")
+@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
+@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
+def delete_cmd(yes: bool, dry_run: bool):
+    """Delete Personas from stdin JSONL."""
+    sys.exit(delete_personas(yes=yes, dry_run=dry_run))
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,309 @@
+#!/usr/bin/env python3
+
+__package__ = "archivebox.cli"
+
+
+import rich_click as click
+
+from archivebox.misc.util import docstring, enforce_types
+
+
+# State Machine ASCII Art Diagrams
+CRAWL_MACHINE_DIAGRAM = """
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                              CrawlMachine                                   │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                             │
+│   ┌─────────────┐                                                           │
+│   │   QUEUED    │◄────────────────┐                                         │
+│   │  (initial)  │                 │                                         │
+│   └──────┬──────┘                 │                                         │
+│          │                        │ tick() unless can_start()               │
+│          │ tick() when            │                                         │
+│          │ can_start()            │                                         │
+│          ▼                        │                                         │
+│   ┌─────────────┐                 │                                         │
+│   │   STARTED   │─────────────────┘                                         │
+│   │             │◄────────────────┐                                         │
+│   │ enter:      │                 │                                         │
+│   │  crawl.run()│                 │ tick() unless is_finished()             │
+│   │  (discover  │                 │                                         │
+│   │   Crawl     │─────────────────┘                                         │
+│   │   hooks)    │                                                           │
+│   └──────┬──────┘                                                           │
+│          │                                                                  │
+│          │ tick() when is_finished()                                        │
+│          ▼                                                                  │
+│   ┌─────────────┐                                                           │
+│   │   SEALED    │                                                           │
+│   │   (final)   │                                                           │
+│   │             │                                                           │
+│   │ enter:      │                                                           │
+│   │  cleanup()  │                                                           │
+│   └─────────────┘                                                           │
+│                                                                             │
+│   Hooks triggered: on_Crawl__* (during STARTED.enter via crawl.run())       │
+│                    on_CrawlEnd__* (during SEALED.enter via cleanup())       │
+└─────────────────────────────────────────────────────────────────────────────┘
+"""
+
+SNAPSHOT_MACHINE_DIAGRAM = """
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                            SnapshotMachine                                  │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                             │
+│   ┌─────────────┐                                                           │
+│   │   QUEUED    │◄────────────────┐                                         │
+│   │  (initial)  │                 │                                         │
+│   └──────┬──────┘                 │                                         │
+│          │                        │ tick() unless can_start()               │
+│          │ tick() when            │                                         │
+│          │ can_start()            │                                         │
+│          ▼                        │                                         │
+│   ┌─────────────┐                 │                                         │
+│   │   STARTED   │─────────────────┘                                         │
+│   │             │◄────────────────┐                                         │
+│   │ enter:      │                 │                                         │
+│   │ snapshot    │                 │ tick() unless is_finished()             │
+│   │  .run()     │                 │                                         │
+│   │ (discover   │─────────────────┘                                         │
+│   │  Snapshot   │                                                           │
+│   │  hooks,     │                                                           │
+│   │  create     │                                                           │
+│   │  pending    │                                                           │
+│   │  results)   │                                                           │
+│   └──────┬──────┘                                                           │
+│          │                                                                  │
+│          │ tick() when is_finished()                                        │
+│          ▼                                                                  │
+│   ┌─────────────┐                                                           │
+│   │   SEALED    │                                                           │
+│   │   (final)   │                                                           │
+│   │             │                                                           │
+│   │ enter:      │                                                           │
+│   │  cleanup()  │                                                           │
+│   └─────────────┘                                                           │
+│                                                                             │
+│   Hooks triggered: on_Snapshot__* (creates ArchiveResults in STARTED.enter) │
+└─────────────────────────────────────────────────────────────────────────────┘
+"""
+
+BINARY_MACHINE_DIAGRAM = """
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                             BinaryMachine                                   │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                             │
+│   ┌─────────────┐                                                           │
+│   │   QUEUED    │◄────────────────┐                                         │
+│   │  (initial)  │                 │                                         │
+│   └──────┬──────┘                 │                                         │
+│          │                        │ tick() unless can_install()             │
+│          │                        │ (stays queued if failed)                │
+│          │ tick() when            │                                         │
+│          │ can_install()          │                                         │
+│          │                        │                                         │
+│          │ on_install() runs      │                                         │
+│          │ during transition:     │                                         │
+│          │  • binary.run()        │                                         │
+│          │    (discover Binary    │                                         │
+│          │     hooks, try each    │                                         │
+│          │     provider until     │                                         │
+│          │     one succeeds)      │                                         │
+│          │  • Sets abspath,       │                                         │
+│          │    version, sha256     │                                         │
+│          │                        │                                         │
+│          │ If install fails:      │                                         │
+│          │  raises exception──────┘                                         │
+│          │  (retry_at bumped)                                               │
+│          │                                                                  │
+│          ▼                                                                  │
+│   ┌─────────────┐                                                           │
+│   │  INSTALLED  │                                                           │
+│   │   (final)   │                                                           │
+│   │             │                                                           │
+│   │ Binary is   │                                                           │
+│   │ ready to    │                                                           │
+│   │ use         │                                                           │
+│   └─────────────┘                                                           │
+│                                                                             │
+│   Hooks triggered: on_Binary__* (provider hooks during transition)          │
+│   Providers tried in sequence until one succeeds: apt, brew, pip, npm, etc. │
+│   Installation is synchronous - no intermediate STARTED state               │
+└─────────────────────────────────────────────────────────────────────────────┘
+"""
+
+
+@enforce_types
+def pluginmap(
+    show_disabled: bool = False,
+    model: str | None = None,
+    quiet: bool = False,
+) -> dict:
+    """
+    Show a map of all state machines and their associated plugin hooks.
+
+    Displays ASCII art diagrams of the core queued model state machines (Crawl,
+    Snapshot, Binary) and lists all auto-detected on_Modelname_xyz hooks
+    that will run for each model's transitions.
+    """
+    from rich.console import Console
+    from rich.table import Table
+    from rich.panel import Panel
+    from rich import box
+
+    from archivebox.hooks import (
+        discover_hooks,
+        is_background_hook,
+        BUILTIN_PLUGINS_DIR,
+        USER_PLUGINS_DIR,
+    )
+
+    console = Console()
+    prnt = console.print
+
+    # Model event types that can have hooks
+    model_events = {
+        "Crawl": {
+            "description": "Hooks run when a Crawl starts (QUEUED→STARTED)",
+            "machine": "CrawlMachine",
+            "diagram": CRAWL_MACHINE_DIAGRAM,
+        },
+        "CrawlEnd": {
+            "description": "Hooks run when a Crawl finishes (STARTED→SEALED)",
+            "machine": "CrawlMachine",
+            "diagram": None,  # Part of CrawlMachine
+        },
+        "Snapshot": {
+            "description": "Hooks run for each Snapshot (creates ArchiveResults)",
+            "machine": "SnapshotMachine",
+            "diagram": SNAPSHOT_MACHINE_DIAGRAM,
+        },
+        "Binary": {
+            "description": "Hooks for installing binary dependencies (providers)",
+            "machine": "BinaryMachine",
+            "diagram": BINARY_MACHINE_DIAGRAM,
+        },
+    }
+
+    # Filter to specific model if requested
+    if model:
+        model = model.title()
+        if model not in model_events:
+            prnt(f'[red]Error: Unknown model "{model}". Available: {", ".join(model_events.keys())}[/red]')
+            return {}
+        model_events = {model: model_events[model]}
+
+    result = {
+        "models": {},
+        "plugins_dir": str(BUILTIN_PLUGINS_DIR),
+        "user_plugins_dir": str(USER_PLUGINS_DIR),
+    }
+
+    if not quiet:
+        prnt()
+        prnt("[bold cyan]ArchiveBox Plugin Map[/bold cyan]")
+        prnt(f"[dim]Built-in plugins: {BUILTIN_PLUGINS_DIR}[/dim]")
+        prnt(f"[dim]User plugins: {USER_PLUGINS_DIR}[/dim]")
+        prnt()
+
+    for event_name, info in model_events.items():
+        # Discover hooks for this event
+        hooks = discover_hooks(event_name, filter_disabled=not show_disabled)
+
+        # Build hook info list
+        hook_infos = []
+        for hook_path in hooks:
+            # Get plugin name from parent directory (e.g., 'wget' from 'plugins/wget/on_Snapshot__06_wget.bg.py')
+            plugin_name = hook_path.parent.name
+            is_bg = is_background_hook(hook_path.name)
+
+            hook_infos.append(
+                {
+                    "path": str(hook_path),
+                    "name": hook_path.name,
+                    "plugin": plugin_name,
+                    "is_background": is_bg,
+                    "extension": hook_path.suffix,
+                },
+            )
+
+        result["models"][event_name] = {
+            "description": info["description"],
+            "machine": info["machine"],
+            "hooks": hook_infos,
+            "hook_count": len(hook_infos),
+        }
+
+        if not quiet:
+            # Show diagram if this model has one
+            if info.get("diagram"):
+                assert info["diagram"] is not None
+                prnt(
+                    Panel(
+                        info["diagram"],
+                        title=f"[bold green]{info['machine']}[/bold green]",
+                        border_style="green",
+                        expand=False,
+                    ),
+                )
+                prnt()
+
+            # Create hooks table
+            table = Table(
+                title=f"[bold yellow]on_{event_name}__* Hooks[/bold yellow] ({len(hooks)} found)",
+                box=box.ROUNDED,
+                show_header=True,
+                header_style="bold magenta",
+            )
+            table.add_column("Plugin", style="cyan", width=20)
+            table.add_column("Hook Name", style="green")
+            table.add_column("BG", justify="center", width=4)
+            table.add_column("Type", justify="center", width=5)
+
+            # Sort lexicographically by hook name
+            sorted_hooks = sorted(hook_infos, key=lambda h: h["name"])
+
+            for hook in sorted_hooks:
+                bg_marker = "[yellow]bg[/yellow]" if hook["is_background"] else ""
+                ext = hook["extension"].lstrip(".")
+                table.add_row(
+                    hook["plugin"],
+                    hook["name"],
+                    bg_marker,
+                    ext,
+                )
+
+            prnt(table)
+            prnt()
+            prnt(f"[dim]{info['description']}[/dim]")
+            prnt()
+
+    # Summary
+    if not quiet:
+        total_hooks = sum(m["hook_count"] for m in result["models"].values())
+        prnt(f"[bold]Total hooks discovered: {total_hooks}[/bold]")
+        prnt()
+        prnt("[dim]Hook naming convention: on_{Model}__{XX}_{description}[.bg].{ext}[/dim]")
+        prnt("[dim]  - XX: Two-digit lexicographic order (00-99)[/dim]")
+        prnt("[dim]  - .bg: Background hook (non-blocking)[/dim]")
+        prnt("[dim]  - ext: py, sh, or js[/dim]")
+        prnt()
+
+    return result
+
+
+@click.command()
+@click.option("--show-disabled", "-a", is_flag=True, help="Show hooks from disabled plugins too")
+@click.option("--model", "-m", type=str, default=None, help="Filter to specific model (Crawl, Snapshot, Binary, CrawlEnd)")
+@click.option("--quiet", "-q", is_flag=True, help="Output JSON only, no ASCII diagrams")
+@docstring(pluginmap.__doc__)
+def main(**kwargs):
+    import json
+
+    result = pluginmap(**kwargs)
+    if kwargs.get("quiet"):
+        print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+
+"""
+archivebox process <action> [--filters]
+
+Manage Process records (system-managed, mostly read-only).
+
+Process records track executions of binaries during extraction.
+They are created automatically by the system and are primarily for debugging.
+
+Actions:
+    list    - List Processes as JSONL (with optional filters)
+
+Examples:
+    # List all processes
+    archivebox process list
+
+    # List processes by binary
+    archivebox process list --binary-name=chrome
+
+    # List recent processes
+    archivebox process list --limit=10
+"""
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox process"
+
+import sys
+
+import rich_click as click
+from rich import print as rprint
+
+from archivebox.cli.cli_utils import apply_filters
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+
+def list_processes(
+    binary_name: str | None = None,
+    machine_id: str | None = None,
+    limit: int | None = None,
+) -> int:
+    """
+    List Processes as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.machine.models import Process
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Process.objects.all().select_related("binary", "machine").order_by("-start_ts")
+
+    # Apply filters
+    filter_kwargs = {}
+    if binary_name:
+        filter_kwargs["binary__name"] = binary_name
+    if machine_id:
+        filter_kwargs["machine_id"] = machine_id
+
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for process in queryset:
+        if is_tty:
+            binary_name_str = process.binary.name if process.binary else "unknown"
+            exit_code = process.exit_code if process.exit_code is not None else "?"
+            status_color = "green" if process.exit_code == 0 else "red" if process.exit_code else "yellow"
+            rprint(f"[{status_color}]exit={exit_code:3}[/{status_color}] [cyan]{binary_name_str:15}[/cyan] [dim]{process.id}[/dim]")
+        else:
+            write_record(process.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} processes[/dim]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+
+@click.group()
+def main():
+    """Manage Process records (read-only, system-managed)."""
+    pass
+
+
+@main.command("list")
+@click.option("--binary-name", "-b", help="Filter by binary name")
+@click.option("--machine-id", "-m", help="Filter by machine ID")
+@click.option("--limit", "-n", type=int, help="Limit number of results")
+def list_cmd(binary_name: str | None, machine_id: str | None, limit: int | None):
+    """List Processes as JSONL."""
+    sys.exit(
+        list_processes(
+            binary_name=binary_name,
+            machine_id=machine_id,
+            limit=limit,
+        ),
+    )
+
+
+if __name__ == "__main__":
+    main()
@@ -1,82 +1,112 @@
 #!/usr/bin/env python3

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox remove'
+__package__ = "archivebox.cli"
+__command__ = "archivebox remove"

-import sys
-import argparse
+import shutil
 from pathlib import Path
-from typing import Optional, List, IO
+from collections.abc import Iterable
+
+import rich_click as click
+
+from django.db.models import QuerySet

-from archivebox.misc.util import docstring
 from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, accept_stdin
-from ..main import remove
+from archivebox.config.constants import CONSTANTS
+from archivebox.config.django import setup_django
+from archivebox.misc.util import enforce_types, docstring
+from archivebox.misc.checks import check_data_folder
+from archivebox.misc.logging_util import (
+    log_list_started,
+    log_list_finished,
+    log_removal_started,
+    log_removal_finished,
+    TimedProgress,
+)


+@enforce_types
+def remove(
+    filter_patterns: Iterable[str] = (),
+    filter_type: str = "exact",
+    snapshots: QuerySet | None = None,
+    after: float | None = None,
+    before: float | None = None,
+    yes: bool = False,
+    delete: bool = False,
+    out_dir: Path = DATA_DIR,
+) -> QuerySet:
+    """Remove the specified URLs from the archive"""
+
+    setup_django()
+    check_data_folder()
+
+    from archivebox.cli.archivebox_search import get_snapshots
+
+    pattern_list = list(filter_patterns)
+
+    log_list_started(pattern_list or None, filter_type)
+    timer = TimedProgress(360, prefix="      ")
+    try:
+        snapshots = get_snapshots(
+            snapshots=snapshots,
+            filter_patterns=pattern_list or None,
+            filter_type=filter_type,
+            after=after,
+            before=before,
+        )
+    finally:
+        timer.end()
+
+    if not snapshots.exists():
+        log_removal_finished(0, 0)
+        raise SystemExit(1)
+
+    log_list_finished(snapshots)
+    log_removal_started(snapshots, yes=yes, delete=delete)
+
+    timer = TimedProgress(360, prefix="      ")
+    try:
+        for snapshot in snapshots:
+            if delete:
+                shutil.rmtree(snapshot.output_dir, ignore_errors=True)
+                legacy_path = CONSTANTS.ARCHIVE_DIR / snapshot.timestamp
+                if legacy_path.is_symlink():
+                    legacy_path.unlink(missing_ok=True)
+    finally:
+        timer.end()
+
+    to_remove = snapshots.count()
+
+    from archivebox.search import flush_search_index
+    from archivebox.core.models import Snapshot
+
+    flush_search_index(snapshots=snapshots)
+    snapshots.delete()
+    all_snapshots = Snapshot.objects.all()
+    log_removal_finished(all_snapshots.count(), to_remove)
+
+    return all_snapshots
+
+
+@click.command()
+@click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
+@click.option("--delete", is_flag=True, help="Delete the archived content and metadata folder in addition to removing from index")
+@click.option("--before", type=float, help="Remove only URLs bookmarked before timestamp")
+@click.option("--after", type=float, help="Remove only URLs bookmarked after timestamp")
+@click.option(
+    "--filter-type",
+    "-f",
+    type=click.Choice(("exact", "substring", "domain", "regex", "tag")),
+    default="exact",
+    help="Type of pattern matching to use when filtering URLs",
+)
+@click.argument("filter_patterns", nargs=-1)
@docstring(remove.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=remove.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        '--yes', # '-y',
-        action='store_true',
-        help='Remove links instantly without prompting to confirm.',
-    )
-    parser.add_argument(
-        '--delete', # '-r',
-        action='store_true',
-        help=(
-            "In addition to removing the link from the index, "
-            "also delete its archived content and metadata folder."
-        ),
-    )
-    parser.add_argument(
-        '--before', #'-b',
-        type=float,
-        help="List only URLs bookmarked before the given timestamp.",
-        default=None,
-    )
-    parser.add_argument(
-        '--after', #'-a',
-        type=float,
-        help="List only URLs bookmarked after the given timestamp.",
-        default=None,
-    )
-    parser.add_argument(
-        '--filter-type',
-        type=str,
-        choices=('exact', 'substring', 'domain', 'regex','tag'),
-        default='exact',
-        help='Type of pattern matching to use when filtering URLs',
-    )
-    parser.add_argument(
-        'filter_patterns',
-        nargs='*',
-        type=str,
-        help='URLs matching this filter pattern will be removed from the index.'
-    )
-    command = parser.parse_args(args or ())
-    
-    filter_str = None
-    if not command.filter_patterns:
-        filter_str = accept_stdin(stdin)
+def main(**kwargs):
+    """Remove the specified URLs from the archive"""
+    remove(**kwargs)

-    remove(
-        filter_str=filter_str,
-        filter_patterns=command.filter_patterns,
-        filter_type=command.filter_type,
-        before=command.before,
-        after=command.after,
-        yes=command.yes,
-        delete=command.delete,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
-    )
-    

-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,359 @@
+#!/usr/bin/env python3
+
+"""
+archivebox run [--daemon] [--crawl-id=...] [--snapshot-id=...] [--binary-id=...]
+
+Unified command for processing queued work on the shared abx-dl bus.
+
+Modes:
+    - With stdin JSONL: Process piped records, exit when complete
+    - Without stdin (TTY): Run the background runner in foreground until killed
+    - --crawl-id: Run the crawl runner for a specific crawl only
+    - --snapshot-id: Run a specific snapshot through its parent crawl
+    - --binary-id: Emit a BinaryEvent for a specific Binary row
+
+Examples:
+    # Run the background runner in foreground
+    archivebox run
+
+    # Run as daemon (don't exit on idle)
+    archivebox run --daemon
+
+    # Process specific records (pipe any JSONL type, exits when done)
+    archivebox snapshot list --status=queued | archivebox run
+    archivebox archiveresult list --status=failed | archivebox run
+    archivebox crawl list --status=queued | archivebox run
+
+    # Mixed types work too
+    cat mixed_records.jsonl | archivebox run
+
+    # Run the crawl runner for a specific crawl
+    archivebox run --crawl-id=019b7e90-04d0-73ed-adec-aad9cfcd863e
+
+    # Run one snapshot from an existing crawl
+    archivebox run --snapshot-id=019b7e90-5a8e-712c-9877-2c70eebe80ad
+
+    # Run one queued binary install directly on the bus
+    archivebox run --binary-id=019b7e90-5a8e-712c-9877-2c70eebe80ad
+"""
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox run"
+
+import sys
+from collections import defaultdict
+
+import rich_click as click
+from rich import print as rprint
+
+
+def process_stdin_records() -> int:
+    """
+    Process JSONL records from stdin.
+
+    Create-or-update behavior:
+    - Records WITHOUT id: Create via Model.from_json(), then queue
+    - Records WITH id: Lookup existing, re-queue for processing
+
+    Outputs JSONL of all processed records (for chaining).
+
+    Handles any record type: Crawl, Snapshot, ArchiveResult.
+    Auto-cascades: Crawl → Snapshots → ArchiveResults.
+
+    Returns exit code (0 = success, 1 = error).
+    """
+    from django.utils import timezone
+
+    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_CRAWL, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT, TYPE_BINARY
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.core.models import Snapshot, ArchiveResult
+    from archivebox.crawls.models import Crawl
+    from archivebox.machine.models import Binary
+    from archivebox.services.runner import run_binary, run_crawl
+
+    records = list(read_stdin())
+    is_tty = sys.stdout.isatty()
+
+    if not records:
+        return 0  # Nothing to process
+
+    created_by_id = get_or_create_system_user_pk()
+    queued_count = 0
+    output_records = []
+    full_crawl_ids: set[str] = set()
+    snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
+    plugin_names_by_crawl: dict[str, set[str]] = defaultdict(set)
+    run_all_plugins_for_crawl: set[str] = set()
+    binary_ids: list[str] = []
+
+    for record in records:
+        record_type = record.get("type", "")
+        record_id = record.get("id")
+
+        try:
+            if record_type == TYPE_CRAWL:
+                if record_id:
+                    # Existing crawl - re-queue
+                    try:
+                        crawl = Crawl.objects.get(id=record_id)
+                    except Crawl.DoesNotExist:
+                        crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
+                else:
+                    # New crawl - create it
+                    crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
+
+                if crawl:
+                    crawl.retry_at = timezone.now()
+                    if crawl.status not in [Crawl.StatusChoices.SEALED]:
+                        crawl.status = Crawl.StatusChoices.QUEUED
+                    crawl.save()
+                    full_crawl_ids.add(str(crawl.id))
+                    run_all_plugins_for_crawl.add(str(crawl.id))
+                    output_records.append(crawl.to_json())
+                    queued_count += 1
+
+            elif record_type == TYPE_SNAPSHOT or (record.get("url") and not record_type):
+                if record_id:
+                    # Existing snapshot - re-queue
+                    try:
+                        snapshot = Snapshot.objects.get(id=record_id)
+                    except Snapshot.DoesNotExist:
+                        snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
+                else:
+                    # New snapshot - create it
+                    snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
+
+                if snapshot:
+                    snapshot.retry_at = timezone.now()
+                    if snapshot.status not in [Snapshot.StatusChoices.SEALED]:
+                        snapshot.status = Snapshot.StatusChoices.QUEUED
+                    snapshot.save()
+                    crawl = snapshot.crawl
+                    crawl.retry_at = timezone.now()
+                    if crawl.status != Crawl.StatusChoices.STARTED:
+                        crawl.status = Crawl.StatusChoices.QUEUED
+                    crawl.save(update_fields=["status", "retry_at", "modified_at"])
+                    crawl_id = str(snapshot.crawl_id)
+                    snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
+                    run_all_plugins_for_crawl.add(crawl_id)
+                    output_records.append(snapshot.to_json())
+                    queued_count += 1
+
+            elif record_type == TYPE_ARCHIVERESULT:
+                if record_id:
+                    # Existing archiveresult - re-queue
+                    try:
+                        archiveresult = ArchiveResult.objects.get(id=record_id)
+                    except ArchiveResult.DoesNotExist:
+                        archiveresult = None
+                else:
+                    archiveresult = None
+
+                snapshot_id = record.get("snapshot_id")
+                plugin_name = record.get("plugin")
+                snapshot = None
+                if archiveresult:
+                    if archiveresult.status in [
+                        ArchiveResult.StatusChoices.FAILED,
+                        ArchiveResult.StatusChoices.SKIPPED,
+                        ArchiveResult.StatusChoices.NORESULTS,
+                        ArchiveResult.StatusChoices.BACKOFF,
+                    ]:
+                        archiveresult.reset_for_retry()
+                    snapshot = archiveresult.snapshot
+                    plugin_name = plugin_name or archiveresult.plugin
+                elif snapshot_id:
+                    try:
+                        snapshot = Snapshot.objects.get(id=snapshot_id)
+                    except Snapshot.DoesNotExist:
+                        snapshot = None
+
+                if snapshot:
+                    snapshot.retry_at = timezone.now()
+                    if snapshot.status != Snapshot.StatusChoices.STARTED:
+                        snapshot.status = Snapshot.StatusChoices.QUEUED
+                    snapshot.save(update_fields=["status", "retry_at", "modified_at"])
+                    crawl = snapshot.crawl
+                    crawl.retry_at = timezone.now()
+                    if crawl.status != Crawl.StatusChoices.STARTED:
+                        crawl.status = Crawl.StatusChoices.QUEUED
+                    crawl.save(update_fields=["status", "retry_at", "modified_at"])
+                    crawl_id = str(snapshot.crawl_id)
+                    snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
+                    if plugin_name:
+                        plugin_names_by_crawl[crawl_id].add(str(plugin_name))
+                    output_records.append(record if not archiveresult else archiveresult.to_json())
+                    queued_count += 1
+
+            elif record_type == TYPE_BINARY:
+                if record_id:
+                    try:
+                        binary = Binary.objects.get(id=record_id)
+                    except Binary.DoesNotExist:
+                        binary = Binary.from_json(record)
+                else:
+                    binary = Binary.from_json(record)
+
+                if binary:
+                    binary.retry_at = timezone.now()
+                    if binary.status != Binary.StatusChoices.INSTALLED:
+                        binary.status = Binary.StatusChoices.QUEUED
+                    binary.save()
+                    binary_ids.append(str(binary.id))
+                    output_records.append(binary.to_json())
+                    queued_count += 1
+
+            else:
+                # Unknown type - pass through
+                output_records.append(record)
+
+        except Exception as e:
+            rprint(f"[yellow]Error processing record: {e}[/yellow]", file=sys.stderr)
+            continue
+
+    # Output all processed records (for chaining)
+    if not is_tty:
+        for rec in output_records:
+            write_record(rec)
+
+    if queued_count == 0:
+        rprint("[yellow]No records to process[/yellow]", file=sys.stderr)
+        return 0
+
+    rprint(f"[blue]Processing {queued_count} records...[/blue]", file=sys.stderr)
+
+    for binary_id in binary_ids:
+        run_binary(binary_id)
+
+    targeted_crawl_ids = full_crawl_ids | set(snapshot_ids_by_crawl)
+    if targeted_crawl_ids:
+        for crawl_id in sorted(targeted_crawl_ids):
+            run_crawl(
+                crawl_id,
+                snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
+                selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
+            )
+    return 0
+
+
+def run_runner(daemon: bool = False) -> int:
+    """
+    Run the background runner loop.
+
+    Args:
+        daemon: Run forever (don't exit when idle)
+
+    Returns exit code (0 = success, 1 = error).
+    """
+    from django.utils import timezone
+    from archivebox.machine.models import Machine, Process
+    from archivebox.services.runner import recover_orphaned_crawls, recover_orphaned_snapshots, run_pending_crawls
+
+    Process.cleanup_stale_running()
+    Process.cleanup_orphaned_workers()
+    recover_orphaned_snapshots()
+    recover_orphaned_crawls()
+    Machine.current()
+    current = Process.current()
+    if current.process_type != Process.TypeChoices.ORCHESTRATOR:
+        current.process_type = Process.TypeChoices.ORCHESTRATOR
+        current.save(update_fields=["process_type", "modified_at"])
+
+    try:
+        run_pending_crawls(daemon=daemon)
+        return 0
+    except KeyboardInterrupt:
+        return 0
+    except Exception as e:
+        rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+        return 1
+    finally:
+        current.refresh_from_db()
+        if current.status != Process.StatusChoices.EXITED:
+            current.status = Process.StatusChoices.EXITED
+            current.ended_at = current.ended_at or timezone.now()
+            current.save(update_fields=["status", "ended_at", "modified_at"])
+
+
+@click.command()
+@click.option("--daemon", "-d", is_flag=True, help="Run forever (don't exit on idle)")
+@click.option("--crawl-id", help="Run the crawl runner for a specific crawl only")
+@click.option("--snapshot-id", help="Run one snapshot through its crawl")
+@click.option("--binary-id", help="Run one queued binary install directly on the bus")
+def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str):
+    """
+    Process queued work.
+
+    Modes:
+    - No args + stdin piped: Process piped JSONL records
+    - No args + TTY: Run the crawl runner for all work
+    - --crawl-id: Run the crawl runner for that crawl only
+    - --snapshot-id: Run one snapshot through its crawl only
+    - --binary-id: Run one queued binary install directly on the bus
+    """
+    if snapshot_id:
+        sys.exit(run_snapshot_worker(snapshot_id))
+
+    if binary_id:
+        try:
+            from archivebox.services.runner import run_binary
+
+            run_binary(binary_id)
+            sys.exit(0)
+        except KeyboardInterrupt:
+            sys.exit(0)
+        except Exception as e:
+            rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+            import traceback
+
+            traceback.print_exc()
+            sys.exit(1)
+
+    if crawl_id:
+        try:
+            from archivebox.services.runner import run_crawl
+
+            run_crawl(crawl_id)
+            sys.exit(0)
+        except KeyboardInterrupt:
+            sys.exit(0)
+        except Exception as e:
+            rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+            import traceback
+
+            traceback.print_exc()
+            sys.exit(1)
+
+    if daemon:
+        if not sys.stdin.isatty():
+            exit_code = process_stdin_records()
+            if exit_code != 0:
+                sys.exit(exit_code)
+        sys.exit(run_runner(daemon=True))
+
+    if not sys.stdin.isatty():
+        sys.exit(process_stdin_records())
+    else:
+        sys.exit(run_runner(daemon=daemon))
+
+
+def run_snapshot_worker(snapshot_id: str) -> int:
+    from archivebox.core.models import Snapshot
+    from archivebox.services.runner import run_crawl
+
+    try:
+        snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)])
+        return 0
+    except KeyboardInterrupt:
+        return 0
+    except Exception as e:
+        rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+        import traceback
+
+        traceback.print_exc()
+        return 1
+
+
+if __name__ == "__main__":
+    main()
@@ -1,116 +1,177 @@
 #!/usr/bin/env python3

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox schedule'
+__package__ = "archivebox.cli"

-import sys
-import argparse
-from pathlib import Path
-from typing import Optional, List, IO
+import rich_click as click
+from rich import print

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
-from ..main import schedule
+from archivebox.misc.util import enforce_types, docstring
+from archivebox.config.common import ARCHIVING_CONFIG


+@enforce_types
+def schedule(
+    add: bool = False,
+    show: bool = False,
+    clear: bool = False,
+    foreground: bool = False,
+    run_all: bool = False,
+    quiet: bool = False,
+    every: str | None = None,
+    tag: str = "",
+    depth: int | str = 0,
+    overwrite: bool = False,
+    update: bool = not ARCHIVING_CONFIG.ONLY_NEW,
+    import_path: str | None = None,
+):
+    """Manage database-backed scheduled crawls processed by the crawl runner."""
+
+    from django.utils import timezone
+
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl, CrawlSchedule
+    from archivebox.crawls.schedule_utils import validate_schedule
+    from archivebox.services.runner import run_pending_crawls
+
+    depth = int(depth)
+    result: dict[str, object] = {
+        "created_schedule_ids": [],
+        "disabled_count": 0,
+        "run_all_enqueued": 0,
+        "active_schedule_ids": [],
+    }
+
+    def _active_schedules():
+        return CrawlSchedule.objects.filter(is_enabled=True).select_related("template").order_by("created_at")
+
+    if clear:
+        disabled_count = CrawlSchedule.objects.filter(is_enabled=True).update(
+            is_enabled=False,
+            modified_at=timezone.now(),
+        )
+        result["disabled_count"] = disabled_count
+        print(f"[green]\\[√] Disabled {disabled_count} scheduled crawl(s).[/green]")
+
+    if every or add:
+        schedule_str = (every or "day").strip()
+        validate_schedule(schedule_str)
+
+        created_by_id = get_or_create_system_user_pk()
+        is_update_schedule = not import_path
+        template_urls = import_path or "archivebox://update"
+        template_label = (f"Scheduled import: {template_urls}" if import_path else "Scheduled ArchiveBox update")[:64]
+        template_notes = (
+            f"Created by archivebox schedule for {template_urls}"
+            if import_path
+            else "Created by archivebox schedule to queue recurring archivebox://update maintenance crawls."
+        )
+
+        template = Crawl.objects.create(
+            urls=template_urls,
+            max_depth=0 if is_update_schedule else depth,
+            tags_str="" if is_update_schedule else tag,
+            label=template_label,
+            notes=template_notes,
+            created_by_id=created_by_id,
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
+            config={
+                "ONLY_NEW": not update,
+                "OVERWRITE": overwrite,
+                "DEPTH": 0 if is_update_schedule else depth,
+                "SCHEDULE_KIND": "update" if is_update_schedule else "crawl",
+            },
+        )
+        crawl_schedule = CrawlSchedule.objects.create(
+            template=template,
+            schedule=schedule_str,
+            is_enabled=True,
+            label=template_label,
+            notes=template_notes,
+            created_by_id=created_by_id,
+        )
+        result["created_schedule_ids"] = [str(crawl_schedule.id)]
+
+        schedule_type = "maintenance update" if is_update_schedule else "crawl"
+        print(f"[green]\\[√] Created scheduled {schedule_type}.[/green]")
+        print(f"    id={crawl_schedule.id}")
+        print(f"    every={crawl_schedule.schedule}")
+        print(f"    next_run={crawl_schedule.next_run_at.isoformat()}")
+        if import_path:
+            print(f"    source={import_path}")
+
+    schedules = list(_active_schedules())
+    result["active_schedule_ids"] = [str(schedule.id) for schedule in schedules]
+
+    if show:
+        if schedules:
+            print(f"[green]\\[*] Active scheduled crawls: {len(schedules)}[/green]")
+            for scheduled_crawl in schedules:
+                template = scheduled_crawl.template
+                print(
+                    f"  - id={scheduled_crawl.id} every={scheduled_crawl.schedule} "
+                    f"next_run={scheduled_crawl.next_run_at.isoformat()} "
+                    f"source={template.urls.splitlines()[0] if template.urls else ''}",
+                )
+        else:
+            print("[yellow]\\[*] No scheduled crawls are enabled.[/yellow]")
+
+    if run_all:
+        enqueued = 0
+        now = timezone.now()
+        for scheduled_crawl in schedules:
+            scheduled_crawl.enqueue(queued_at=now)
+            enqueued += 1
+        result["run_all_enqueued"] = enqueued
+        print(f"[green]\\[*] Enqueued {enqueued} scheduled crawl(s) immediately.[/green]")
+        if enqueued:
+            print(
+                "[yellow]\\[*] Start `archivebox server`, `archivebox run --daemon`, or `archivebox schedule --foreground` to process the queued crawls.[/yellow]",
+            )
+
+    if foreground:
+        print(
+            "[green]\\[*] Starting global crawl runner in foreground mode. It will materialize scheduled crawls and process queued work.[/green]",
+        )
+        run_pending_crawls(daemon=True)
+
+    if quiet:
+        return result
+
+    if not any((every, add, show, clear, foreground, run_all)):
+        if schedules:
+            print("[green]\\[*] Active scheduled crawls:[/green]")
+            for scheduled_crawl in schedules:
+                print(f"  - {scheduled_crawl.id} every={scheduled_crawl.schedule} next_run={scheduled_crawl.next_run_at.isoformat()}")
+        else:
+            print("[yellow]\\[*] No scheduled crawls are enabled.[/yellow]")
+
+    return result
+
+
+@click.command()
+@click.option("--quiet", "-q", is_flag=True, help="Return structured results without extra summary output")
+@click.option("--add", is_flag=True, help="Create a new scheduled crawl")
+@click.option("--every", type=str, help='Run on an alias like daily/weekly/monthly or a cron expression such as "0 */6 * * *"')
+@click.option("--tag", "-t", default="", help="Comma-separated tags to apply to scheduled crawl snapshots")
+@click.option(
+    "--depth",
+    type=click.Choice([str(i) for i in range(5)]),
+    default="0",
+    help="Recursively archive linked pages up to N hops away",
+)
+@click.option("--overwrite", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
+@click.option("--update", is_flag=True, help="Retry previously failed/skipped URLs when scheduled crawls run")
+@click.option("--clear", is_flag=True, help="Disable all currently enabled schedules")
+@click.option("--show", is_flag=True, help="Print all currently enabled schedules")
+@click.option("--foreground", "-f", is_flag=True, help="Run the global crawl runner in the foreground (no crontab required)")
+@click.option("--run-all", is_flag=True, help="Enqueue all enabled schedules immediately and process them once")
+@click.argument("import_path", required=False)
@docstring(schedule.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=schedule.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        '--quiet', '-q',
-        action='store_true',
-        help=("Don't warn about storage space."),
-    )
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-        '--add', # '-a',
-        action='store_true',
-        help='Add a new scheduled ArchiveBox update job to cron',
-    )
-    parser.add_argument(
-        '--every', # '-e',
-        type=str,
-        default=None,
-        help='Run ArchiveBox once every [timeperiod] (hour/day/month/year or cron format e.g. "0 0 * * *")',
-    )
-    parser.add_argument(
-        '--tag', '-t',
-        type=str,
-        default='',
-        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
-    )
-    parser.add_argument(
-        '--depth', # '-d',
-        type=int,
-        choices=[0, 1],
-        default=0,
-        help='Depth to archive to [0] or 1, see "add" command help for more info',
-    )
-    parser.add_argument(
-        '--overwrite',
-        action='store_true',
-        help='Re-archive any URLs that have been previously archived, overwriting existing Snapshots',
-    )
-    parser.add_argument(
-        '--update',
-        action='store_true',
-        help='Re-pull any URLs that have been previously added, as needed to fill missing ArchiveResults',
-    )
-    group.add_argument(
-        '--clear', # '-c'
-        action='store_true',
-        help=("Stop all ArchiveBox scheduled runs (remove cron jobs)"),
-    )
-    group.add_argument(
-        '--show', # '-s'
-        action='store_true',
-        help=("Print a list of currently active ArchiveBox cron jobs"),
-    )
-    group.add_argument(
-        '--foreground', '-f',
-        action='store_true',
-        help=("Launch ArchiveBox scheduler as a long-running foreground task "
-              "instead of using cron."),
-    )
-    group.add_argument(
-        '--run-all', # '-a',
-        action='store_true',
-        help=("Run all the scheduled jobs once immediately, independent of "
-              "their configured schedules, can be used together with --foreground"),
-    )
-    parser.add_argument(
-        'import_path',
-        nargs='?',
-        type=str,
-        default=None,
-        help=("Check this path and import any new links on every run "
-              "(can be either local file or remote URL)"),
-    )
-    command = parser.parse_args(args or ())
-    reject_stdin(__command__, stdin)
-
-    schedule(
-        add=command.add,
-        show=command.show,
-        clear=command.clear,
-        foreground=command.foreground,
-        run_all=command.run_all,
-        quiet=command.quiet,
-        every=command.every,
-        tag=command.tag,
-        depth=command.depth,
-        overwrite=command.overwrite,
-        update=command.update,
-        import_path=command.import_path,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
-    )
+def main(**kwargs):
+    """Manage database-backed scheduled crawls processed by the crawl runner."""
+    schedule(**kwargs)


-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,257 @@
+#!/usr/bin/env python3
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox search"
+
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+from collections.abc import Callable
+
+import rich_click as click
+
+from django.db.models import Q, QuerySet
+
+from archivebox.config import DATA_DIR
+from archivebox.misc.logging import stderr
+from archivebox.misc.util import enforce_types, docstring
+
+if TYPE_CHECKING:
+    from archivebox.core.models import Snapshot
+
+# Filter types for URL matching
+LINK_FILTERS: dict[str, Callable[[str], Q]] = {
+    "exact": lambda pattern: Q(url=pattern),
+    "substring": lambda pattern: Q(url__icontains=pattern),
+    "regex": lambda pattern: Q(url__iregex=pattern),
+    "domain": lambda pattern: (
+        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
+    ),
+    "tag": lambda pattern: Q(tags__name=pattern),
+    "timestamp": lambda pattern: Q(timestamp=pattern),
+}
+
+STATUS_CHOICES = ["indexed", "archived", "unarchived"]
+
+
+def _apply_pattern_filters(
+    snapshots: QuerySet["Snapshot", "Snapshot"],
+    filter_patterns: list[str],
+    filter_type: str,
+) -> QuerySet["Snapshot", "Snapshot"]:
+    filter_builder = LINK_FILTERS.get(filter_type)
+    if filter_builder is None:
+        stderr()
+        stderr(f"[X] Got invalid pattern for --filter-type={filter_type}", color="red")
+        raise SystemExit(2)
+
+    query = Q()
+    for pattern in filter_patterns:
+        query |= filter_builder(pattern)
+    return snapshots.filter(query)
+
+
+def _snapshots_to_json(
+    snapshots: QuerySet["Snapshot", "Snapshot"],
+    *,
+    with_headers: bool,
+) -> str:
+    from datetime import datetime, timezone as tz
+
+    from archivebox.config import VERSION
+    from archivebox.config.common import SERVER_CONFIG
+    from archivebox.misc.util import to_json
+
+    main_index_header = (
+        {
+            "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
+            "schema": "archivebox.index.json",
+            "copyright_info": SERVER_CONFIG.FOOTER_INFO,
+            "meta": {
+                "project": "ArchiveBox",
+                "version": VERSION,
+                "git_sha": VERSION,
+                "website": "https://ArchiveBox.io",
+                "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
+                "source": "https://github.com/ArchiveBox/ArchiveBox",
+                "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
+                "dependencies": {},
+            },
+        }
+        if with_headers
+        else {}
+    )
+
+    snapshot_dicts = [snapshot.to_dict(extended=True) for snapshot in snapshots.iterator(chunk_size=500)]
+    output: dict[str, object] | list[dict[str, object]]
+    if with_headers:
+        output = {
+            **main_index_header,
+            "num_links": len(snapshot_dicts),
+            "updated": datetime.now(tz.utc),
+            "last_run_cmd": sys.argv,
+            "links": snapshot_dicts,
+        }
+    else:
+        output = snapshot_dicts
+
+    return to_json(output, indent=4, sort_keys=True)
+
+
+def _snapshots_to_csv(
+    snapshots: QuerySet["Snapshot", "Snapshot"],
+    *,
+    cols: list[str],
+    with_headers: bool,
+) -> str:
+    header = ",".join(cols) if with_headers else ""
+    rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.iterator(chunk_size=500)]
+    return "\n".join((header, *rows))
+
+
+def _snapshots_to_html(
+    snapshots: QuerySet["Snapshot", "Snapshot"],
+    *,
+    with_headers: bool,
+) -> str:
+    from datetime import datetime, timezone as tz
+
+    from django.template.loader import render_to_string
+
+    from archivebox.config import VERSION
+    from archivebox.config.common import SERVER_CONFIG
+    from archivebox.config.version import get_COMMIT_HASH
+
+    template = "static_index.html" if with_headers else "minimal_index.html"
+    snapshot_list = list(snapshots.iterator(chunk_size=500))
+
+    return render_to_string(
+        template,
+        {
+            "version": VERSION,
+            "git_sha": get_COMMIT_HASH() or VERSION,
+            "num_links": str(len(snapshot_list)),
+            "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"),
+            "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"),
+            "links": snapshot_list,
+            "FOOTER_INFO": SERVER_CONFIG.FOOTER_INFO,
+        },
+    )
+
+
+def get_snapshots(
+    snapshots: QuerySet["Snapshot", "Snapshot"] | None = None,
+    filter_patterns: list[str] | None = None,
+    filter_type: str = "substring",
+    after: float | None = None,
+    before: float | None = None,
+    out_dir: Path = DATA_DIR,
+) -> QuerySet["Snapshot", "Snapshot"]:
+    """Filter and return Snapshots matching the given criteria."""
+    from archivebox.core.models import Snapshot
+
+    if snapshots is not None:
+        result = snapshots
+    else:
+        result = Snapshot.objects.all()
+
+    if after is not None:
+        result = result.filter(timestamp__gte=after)
+    if before is not None:
+        result = result.filter(timestamp__lt=before)
+    if filter_patterns:
+        result = _apply_pattern_filters(result, filter_patterns, filter_type)
+
+    # Prefetch crawl relationship to avoid N+1 queries when accessing output_dir
+    result = result.select_related("crawl", "crawl__created_by")
+
+    if not result.exists():
+        stderr("[!] No Snapshots matched your filters:", filter_patterns, f"({filter_type})", color="lightyellow")
+
+    return result
+
+
+@enforce_types
+def search(
+    filter_patterns: list[str] | None = None,
+    filter_type: str = "substring",
+    status: str = "indexed",
+    before: float | None = None,
+    after: float | None = None,
+    sort: str | None = None,
+    json: bool = False,
+    html: bool = False,
+    csv: str | None = None,
+    with_headers: bool = False,
+):
+    """List, filter, and export information about archive entries"""
+
+    if with_headers and not (json or html or csv):
+        stderr("[X] --with-headers requires --json, --html or --csv\n", color="red")
+        raise SystemExit(2)
+
+    # Query DB directly - no filesystem scanning
+    snapshots = get_snapshots(
+        filter_patterns=list(filter_patterns) if filter_patterns else None,
+        filter_type=filter_type,
+        before=before,
+        after=after,
+    )
+
+    # Apply status filter
+    if status == "archived":
+        snapshots = snapshots.filter(downloaded_at__isnull=False)
+    elif status == "unarchived":
+        snapshots = snapshots.filter(downloaded_at__isnull=True)
+    # 'indexed' = all snapshots (no filter)
+
+    if sort:
+        snapshots = snapshots.order_by(sort)
+
+    # Export to requested format
+    if json:
+        output = _snapshots_to_json(snapshots, with_headers=with_headers)
+    elif html:
+        output = _snapshots_to_html(snapshots, with_headers=with_headers)
+    elif csv:
+        output = _snapshots_to_csv(snapshots, cols=csv.split(","), with_headers=with_headers)
+    else:
+        from archivebox.misc.logging_util import printable_folders
+
+        # Convert to dict for printable_folders
+        folders: dict[str, Snapshot | None] = {str(snapshot.output_dir): snapshot for snapshot in snapshots}
+        output = printable_folders(folders, with_headers)
+
+    # Structured exports must be written directly to stdout.
+    # rich.print() reflows long lines to console width, which corrupts JSON/CSV/HTML output.
+    sys.stdout.write(output)
+    if not output.endswith("\n"):
+        sys.stdout.write("\n")
+    return output
+
+
+@click.command()
+@click.option(
+    "--filter-type",
+    "-f",
+    type=click.Choice(["search", *LINK_FILTERS.keys()]),
+    default="substring",
+    help="Pattern matching type for filtering URLs",
+)
+@click.option("--status", "-s", type=click.Choice(STATUS_CHOICES), default="indexed", help="List snapshots with the given status")
+@click.option("--before", "-b", type=float, help="List snapshots bookmarked before the given UNIX timestamp")
+@click.option("--after", "-a", type=float, help="List snapshots bookmarked after the given UNIX timestamp")
+@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
+@click.option("--json", "-J", is_flag=True, help="Print output in JSON format")
+@click.option("--html", "-M", is_flag=True, help="Print output in HTML format (suitable for viewing statically without a server)")
+@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: created_at,url,title")
+@click.option("--with-headers", "-H", is_flag=True, help="Include extra CSV/HTML headers in the output")
+@click.help_option("--help", "-h")
+@click.argument("filter_patterns", nargs=-1)
+@docstring(search.__doc__)
+def main(**kwargs):
+    return search(**kwargs)
+
+
+if __name__ == "__main__":
+    main()
@@ -1,83 +1,235 @@
 #!/usr/bin/env python3

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox server'
+__package__ = "archivebox.cli"

+from collections.abc import Iterable
 import sys
-import argparse
-from pathlib import Path
-from typing import Optional, List, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
+import rich_click as click
+from rich import print
+
+from archivebox.misc.util import docstring, enforce_types
 from archivebox.config.common import SERVER_CONFIG
-from ..logging_util import SmartFormatter, reject_stdin
-from ..main import server

+
+def stop_existing_background_runner(*, machine, process_model, supervisor=None, stop_worker_fn=None, log=print) -> int:
+    """Stop any existing orchestrator process so the server can take ownership."""
+    process_model.cleanup_stale_running(machine=machine)
+    process_model.cleanup_orphaned_workers()
+
+    running_runners = list(
+        process_model.objects.filter(
+            machine=machine,
+            status=process_model.StatusChoices.RUNNING,
+            process_type=process_model.TypeChoices.ORCHESTRATOR,
+        ).order_by("created_at"),
+    )
+
+    if not running_runners:
+        return 0
+
+    log("[yellow][*] Stopping existing ArchiveBox background runner...[/yellow]")
+
+    if supervisor is not None and stop_worker_fn is not None:
+        for worker_name in ("worker_runner", "worker_runner_watch"):
+            try:
+                stop_worker_fn(supervisor, worker_name)
+            except Exception:
+                pass
+
+    for proc in running_runners:
+        try:
+            proc.kill_tree(graceful_timeout=2.0)
+        except Exception:
+            try:
+                proc.terminate(graceful_timeout=2.0)
+            except Exception:
+                pass
+
+    process_model.cleanup_stale_running(machine=machine)
+    return len(running_runners)
+
+
+def _read_supervisor_worker_command(worker_name: str) -> str:
+    from archivebox.workers.supervisord_util import WORKERS_DIR_NAME, get_sock_file
+
+    worker_conf = get_sock_file().parent / WORKERS_DIR_NAME / f"{worker_name}.conf"
+    if not worker_conf.exists():
+        return ""
+
+    for line in worker_conf.read_text().splitlines():
+        if line.startswith("command="):
+            return line.removeprefix("command=").strip()
+    return ""
+
+
+def _worker_command_matches_bind(command: str, host: str, port: str) -> bool:
+    if not command:
+        return False
+    return f"{host}:{port}" in command or (f"--bind={host}" in command and f"--port={port}" in command)
+
+
+def stop_existing_server_workers(*, supervisor, stop_worker_fn, host: str, port: str, log=print) -> int:
+    """Stop existing ArchiveBox web workers if they already own the requested bind."""
+    stopped = 0
+
+    for worker_name in ("worker_runserver", "worker_daphne"):
+        try:
+            proc = supervisor.getProcessInfo(worker_name) if supervisor else None
+        except Exception:
+            proc = None
+        if not isinstance(proc, dict) or proc.get("statename") != "RUNNING":
+            continue
+
+        command = _read_supervisor_worker_command(worker_name)
+        if not _worker_command_matches_bind(command, host, port):
+            continue
+
+        if stopped == 0:
+            log("[yellow][*] Taking over existing ArchiveBox web server on same port...[/yellow]")
+        stop_worker_fn(supervisor, worker_name)
+        stopped += 1
+
+    return stopped
+
+
+@enforce_types
+def server(
+    runserver_args: Iterable[str] = (SERVER_CONFIG.BIND_ADDR,),
+    reload: bool = False,
+    init: bool = False,
+    debug: bool = False,
+    daemonize: bool = False,
+    nothreading: bool = False,
+) -> None:
+    """Run the ArchiveBox HTTP server"""
+
+    runserver_args = list(runserver_args)
+
+    if init:
+        from archivebox.cli.archivebox_init import init as archivebox_init
+
+        archivebox_init(quick=True)
+        print()
+
+    from archivebox.misc.checks import check_data_folder
+
+    check_data_folder()
+
+    from archivebox.config.common import SHELL_CONFIG
+
+    run_in_debug = SHELL_CONFIG.DEBUG or debug or reload
+    if debug or reload:
+        SHELL_CONFIG.DEBUG = True
+
+    from django.contrib.auth.models import User
+
+    if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
+        print()
+        print(
+            "[violet]Hint:[/violet] To create an [bold]admin username & password[/bold] for the [deep_sky_blue3][underline][link=http://{host}:{port}/admin]Admin UI[/link][/underline][/deep_sky_blue3], run:",
+        )
+        print("      [green]archivebox manage createsuperuser[/green]")
+        print()
+
+    host = "127.0.0.1"
+    port = "8000"
+
+    try:
+        host_and_port = [arg for arg in runserver_args if arg.replace(".", "").replace(":", "").isdigit()][0]
+        if ":" in host_and_port:
+            host, port = host_and_port.split(":")
+        else:
+            if "." in host_and_port:
+                host = host_and_port
+            else:
+                port = host_and_port
+    except IndexError:
+        pass
+
+    from archivebox.workers.supervisord_util import (
+        get_existing_supervisord_process,
+        get_worker,
+        stop_worker,
+        start_server_workers,
+        is_port_in_use,
+    )
+    from archivebox.machine.models import Machine, Process
+
+    machine = Machine.current()
+    supervisor = get_existing_supervisord_process()
+    stop_existing_background_runner(
+        machine=machine,
+        process_model=Process,
+        supervisor=supervisor,
+        stop_worker_fn=stop_worker,
+    )
+    if supervisor:
+        stop_existing_server_workers(
+            supervisor=supervisor,
+            stop_worker_fn=stop_worker,
+            host=host,
+            port=port,
+        )
+
+    # Check if port is already in use
+    if is_port_in_use(host, int(port)):
+        print(f"[red][X] Error: Port {port} is already in use[/red]")
+        print(f"    Another process (possibly daphne or runserver) is already listening on {host}:{port}")
+        print("    Stop the conflicting process or choose a different port")
+        sys.exit(1)
+
+    supervisor = get_existing_supervisord_process()
+    if supervisor:
+        server_worker_name = "worker_runserver" if run_in_debug else "worker_daphne"
+        server_proc = get_worker(supervisor, server_worker_name)
+        server_state = server_proc.get("statename") if isinstance(server_proc, dict) else None
+        if server_state == "RUNNING":
+            runner_proc = get_worker(supervisor, "worker_runner")
+            runner_watch_proc = get_worker(supervisor, "worker_runner_watch")
+            runner_state = runner_proc.get("statename") if isinstance(runner_proc, dict) else None
+            runner_watch_state = runner_watch_proc.get("statename") if isinstance(runner_watch_proc, dict) else None
+            print("[red][X] Error: ArchiveBox server is already running[/red]")
+            print(
+                f"    [green]√[/green] Web server ({server_worker_name}) is RUNNING on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
+            )
+            if runner_state == "RUNNING":
+                print("    [green]√[/green] Background runner (worker_runner) is RUNNING")
+            if runner_watch_state == "RUNNING":
+                print("    [green]√[/green] Reload watcher (worker_runner_watch) is RUNNING")
+            print()
+            print("[yellow]To stop the existing server, run:[/yellow]")
+            print('    pkill -f "archivebox server"')
+            print("    pkill -f supervisord")
+            sys.exit(1)
+
+    if run_in_debug:
+        print("[green][+] Starting ArchiveBox webserver in DEBUG mode...[/green]")
+    else:
+        print("[green][+] Starting ArchiveBox webserver...[/green]")
+    print(
+        f"    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
+    )
+    print(
+        f"    [green]>[/green] Log in to ArchiveBox Admin UI on [deep_sky_blue3][link=http://{host}:{port}/admin]http://{host}:{port}/admin[/link][/deep_sky_blue3]",
+    )
+    print("    > Writing ArchiveBox error log to ./logs/errors.log")
+    print()
+    start_server_workers(host=host, port=port, daemonize=daemonize, debug=run_in_debug, reload=reload, nothreading=nothreading)
+    print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")
+
+
+@click.command()
+@click.argument("runserver_args", nargs=-1)
+@click.option("--reload", is_flag=True, help="Enable auto-reloading when code or templates change")
+@click.option("--debug", is_flag=True, help="Enable DEBUG=True mode with more verbose errors")
+@click.option("--nothreading", is_flag=True, help="Force runserver to run in single-threaded mode")
+@click.option("--init", is_flag=True, help="Run a full archivebox init/upgrade before starting the server")
+@click.option("--daemonize", is_flag=True, help="Run the server in the background as a daemon")
@docstring(server.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=server.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        'runserver_args',
-        nargs='*',
-        type=str,
-        default=[SERVER_CONFIG.BIND_ADDR],
-        help='Arguments to pass to Django runserver'
-    )
-    parser.add_argument(
-        '--reload',
-        action='store_true',
-        help='Enable auto-reloading when code or templates change',
-    )
-    parser.add_argument(
-        '--debug',
-        action='store_true',
-        help='Enable DEBUG=True mode with more verbose errors',
-    )
-    parser.add_argument(
-        '--nothreading',
-        action='store_true',
-        help='Force runserver to run in single-threaded mode',
-    )
-    parser.add_argument(
-        '--init',
-        action='store_true',
-        help='Run a full archivebox init/upgrade before starting the server',
-    )
-    parser.add_argument(
-        '--quick-init', '-i',
-        action='store_true',
-        help='Run quick archivebox init/upgrade before starting the server',
-    )
-    parser.add_argument(
-        '--createsuperuser',
-        action='store_true',
-        help='Run archivebox manage createsuperuser before starting the server',
-    )
-    parser.add_argument(
-        '--daemonize',
-        action='store_true',
-        help='Run the server in the background as a daemon',
-    )
-    command = parser.parse_args(args or ())
-    reject_stdin(__command__, stdin)
-    
-    server(
-        runserver_args=command.runserver_args + (['--nothreading'] if command.nothreading else []),
-        reload=command.reload,
-        debug=command.debug,
-        init=command.init,
-        quick_init=command.quick_init,
-        createsuperuser=command.createsuperuser,
-        daemonize=command.daemonize,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
-    )
+def main(**kwargs):
+    server(**kwargs)


-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+if __name__ == "__main__":
+    main()
@@ -1,34 +1,28 @@
 #!/usr/bin/env python3

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox shell'
+__package__ = "archivebox.cli"

-import sys
-import argparse
-from pathlib import Path
-from typing import Optional, List, IO
+from collections.abc import Iterable
+
+import rich_click as click

 from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
-from ..main import shell


+def shell(args: Iterable[str] = ()) -> None:
+    """Enter an interactive ArchiveBox Django shell"""
+
+    from django.core.management import call_command
+
+    call_command("shell_plus", *args)
+
+
+@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
+@click.argument("args", nargs=-1)
@docstring(shell.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=shell.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.parse_args(args or ())
-    reject_stdin(__command__, stdin)
-    
-    shell(
-        out_dir=Path(pwd) if pwd else DATA_DIR,
-    )
-    
+def main(args: Iterable[str] = ()) -> None:
+    shell(args=args)

-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,497 @@
+#!/usr/bin/env python3
+
+"""
+archivebox snapshot <action> [args...] [--filters]
+
+Manage Snapshot records.
+
+Actions:
+    create  - Create Snapshots from URLs or Crawl JSONL
+    list    - List Snapshots as JSONL (with optional filters)
+    update  - Update Snapshots from stdin JSONL
+    delete  - Delete Snapshots from stdin JSONL
+
+Examples:
+    # Create
+    archivebox snapshot create https://example.com --tag=news
+    archivebox crawl create https://example.com | archivebox snapshot create
+
+    # List with filters
+    archivebox snapshot list --status=queued
+    archivebox snapshot list --url__icontains=example.com
+
+    # Update
+    archivebox snapshot list --tag=old | archivebox snapshot update --tag=new
+
+    # Delete
+    archivebox snapshot list --url__icontains=spam.com | archivebox snapshot delete --yes
+"""
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox snapshot"
+
+import sys
+from collections.abc import Iterable
+
+import rich_click as click
+from rich import print as rprint
+from django.db.models import Q, Sum
+from django.db.models.functions import Coalesce
+
+from archivebox.cli.cli_utils import apply_filters
+
+
+# =============================================================================
+# CREATE
+# =============================================================================
+
+
+def create_snapshots(
+    urls: Iterable[str],
+    tag: str = "",
+    status: str = "queued",
+    depth: int = 0,
+    created_by_id: int | None = None,
+) -> int:
+    """
+    Create Snapshots from URLs or stdin JSONL (Crawl or Snapshot records).
+    Pass-through: Records that are not Crawl/Snapshot/URL are output unchanged.
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
+    from archivebox.misc.jsonl import (
+        read_args_or_stdin,
+        write_record,
+        TYPE_SNAPSHOT,
+        TYPE_CRAWL,
+    )
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.core.models import Snapshot
+    from archivebox.crawls.models import Crawl
+
+    created_by_id = created_by_id or get_or_create_system_user_pk()
+    is_tty = sys.stdout.isatty()
+
+    # Collect all input records
+    records = list(read_args_or_stdin(urls))
+
+    if not records:
+        rprint("[yellow]No URLs or Crawls provided. Pass URLs as arguments or via stdin.[/yellow]", file=sys.stderr)
+        return 1
+
+    # Process each record - handle Crawls and plain URLs/Snapshots
+    created_snapshots = []
+    pass_through_count = 0
+
+    for record in records:
+        record_type = record.get("type", "")
+
+        try:
+            if record_type == TYPE_CRAWL:
+                # Pass through the Crawl record itself first
+                if not is_tty:
+                    write_record(record)
+
+                # Input is a Crawl - get or create it, then create Snapshots for its URLs
+                crawl = None
+                crawl_id = record.get("id")
+                if crawl_id:
+                    try:
+                        crawl = Crawl.objects.get(id=crawl_id)
+                    except Crawl.DoesNotExist:
+                        crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
+                else:
+                    crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
+
+                if not crawl:
+                    continue
+
+                # Create snapshots for each URL in the crawl
+                for url in crawl.get_urls_list():
+                    merged_tags = crawl.tags_str
+                    if tag:
+                        merged_tags = f"{merged_tags},{tag}" if merged_tags else tag
+                    snapshot_record = {
+                        "url": url,
+                        "tags": merged_tags,
+                        "crawl_id": str(crawl.id),
+                        "depth": depth,
+                        "status": status,
+                    }
+                    snapshot = Snapshot.from_json(snapshot_record, overrides={"created_by_id": created_by_id})
+                    if snapshot:
+                        created_snapshots.append(snapshot)
+                        if not is_tty:
+                            write_record(snapshot.to_json())
+
+            elif record_type == TYPE_SNAPSHOT or record.get("url"):
+                # Input is a Snapshot or plain URL
+                if tag and not record.get("tags"):
+                    record["tags"] = tag
+                if status:
+                    record["status"] = status
+                record["depth"] = record.get("depth", depth)
+
+                snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
+                if snapshot:
+                    created_snapshots.append(snapshot)
+                    if not is_tty:
+                        write_record(snapshot.to_json())
+
+            else:
+                # Pass-through: output records we don't handle
+                if not is_tty:
+                    write_record(record)
+                pass_through_count += 1
+
+        except Exception as e:
+            rprint(f"[red]Error creating snapshot: {e}[/red]", file=sys.stderr)
+            continue
+
+    if not created_snapshots:
+        if pass_through_count > 0:
+            rprint(f"[dim]Passed through {pass_through_count} records, no new snapshots[/dim]", file=sys.stderr)
+            return 0
+        rprint("[red]No snapshots created[/red]", file=sys.stderr)
+        return 1
+
+    rprint(f"[green]Created {len(created_snapshots)} snapshots[/green]", file=sys.stderr)
+
+    if is_tty:
+        for snapshot in created_snapshots:
+            rprint(f"  [dim]{snapshot.id}[/dim] {snapshot.url[:60]}", file=sys.stderr)
+
+    return 0
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+
+def list_snapshots(
+    status: str | None = None,
+    url__icontains: str | None = None,
+    url__istartswith: str | None = None,
+    tag: str | None = None,
+    crawl_id: str | None = None,
+    limit: int | None = None,
+    sort: str | None = None,
+    csv: str | None = None,
+    with_headers: bool = False,
+    search: str | None = None,
+    query: str | None = None,
+) -> int:
+    """
+    List Snapshots as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.core.models import Snapshot
+    from archivebox.search import (
+        get_default_search_mode,
+        get_search_mode,
+        prioritize_metadata_matches,
+        query_search_index,
+    )
+
+    if with_headers and not csv:
+        rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr)
+        return 2
+
+    is_tty = sys.stdout.isatty() and not csv
+
+    queryset = Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)).order_by("-created_at")
+
+    # Apply filters
+    filter_kwargs = {
+        "status": status,
+        "url__icontains": url__icontains,
+        "url__istartswith": url__istartswith,
+        "crawl_id": crawl_id,
+    }
+    queryset = apply_filters(queryset, filter_kwargs)
+
+    # Tag filter requires special handling (M2M)
+    if tag:
+        queryset = queryset.filter(tags__name__iexact=tag)
+
+    query = (query or "").strip()
+    if query:
+        metadata_qs = queryset.filter(
+            Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query),
+        )
+        requested_search_mode = (search or "").strip().lower()
+        if requested_search_mode == "content":
+            requested_search_mode = "contents"
+        search_mode = get_default_search_mode() if not requested_search_mode else get_search_mode(requested_search_mode)
+
+        if search_mode == "meta":
+            queryset = metadata_qs
+        else:
+            try:
+                deep_qsearch = None
+                if search_mode == "deep":
+                    qsearch = query_search_index(query, search_mode="contents")
+                    deep_qsearch = query_search_index(query, search_mode="deep")
+                else:
+                    qsearch = query_search_index(query, search_mode=search_mode)
+                queryset = prioritize_metadata_matches(
+                    queryset,
+                    metadata_qs,
+                    qsearch,
+                    deep_queryset=deep_qsearch,
+                    ordering=("-created_at",) if not sort else None,
+                )
+            except Exception as err:
+                rprint(
+                    f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
+                    file=sys.stderr,
+                )
+                queryset = metadata_qs
+
+    if sort:
+        queryset = queryset.order_by(sort)
+    if limit:
+        queryset = queryset[:limit]
+
+    count = 0
+    if csv:
+        cols = [col.strip() for col in csv.split(",") if col.strip()]
+        if not cols:
+            rprint("[red]No CSV columns provided[/red]", file=sys.stderr)
+            return 2
+        rows: list[str] = []
+        if with_headers:
+            rows.append(",".join(cols))
+        for snapshot in queryset.iterator(chunk_size=500):
+            rows.append(snapshot.to_csv(cols=cols, separator=","))
+            count += 1
+        output = "\n".join(rows)
+        if output:
+            sys.stdout.write(output)
+            if not output.endswith("\n"):
+                sys.stdout.write("\n")
+        rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    for snapshot in queryset:
+        if is_tty:
+            status_color = {
+                "queued": "yellow",
+                "started": "blue",
+                "sealed": "green",
+            }.get(snapshot.status, "dim")
+            rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
+        else:
+            write_record(snapshot.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# UPDATE
+# =============================================================================
+
+
+def update_snapshots(
+    status: str | None = None,
+    tag: str | None = None,
+) -> int:
+    """
+    Update Snapshots from stdin JSONL.
+
+    Reads Snapshot records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from django.utils import timezone
+
+    from archivebox.misc.jsonl import read_stdin, write_record
+    from archivebox.core.models import Snapshot
+
+    is_tty = sys.stdout.isatty()
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        snapshot_id = record.get("id")
+        if not snapshot_id:
+            continue
+
+        try:
+            snapshot = Snapshot.objects.get(id=snapshot_id)
+
+            # Apply updates from CLI flags (override stdin values)
+            if status:
+                snapshot.status = status
+                snapshot.retry_at = timezone.now()
+            if tag:
+                # Add tag to existing tags
+                snapshot.save()  # Ensure saved before M2M
+                from archivebox.core.models import Tag
+
+                tag_obj, _ = Tag.objects.get_or_create(name=tag)
+                snapshot.tags.add(tag_obj)
+
+            snapshot.save()
+            updated_count += 1
+
+            if not is_tty:
+                write_record(snapshot.to_json())
+
+        except Snapshot.DoesNotExist:
+            rprint(f"[yellow]Snapshot not found: {snapshot_id}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} snapshots[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# DELETE
+# =============================================================================
+
+
+def delete_snapshots(yes: bool = False, dry_run: bool = False) -> int:
+    """
+    Delete Snapshots from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
+    from archivebox.core.models import Snapshot
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    snapshot_ids = [r.get("id") for r in records if r.get("id")]
+
+    if not snapshot_ids:
+        rprint("[yellow]No valid snapshot IDs in input[/yellow]", file=sys.stderr)
+        return 1
+
+    snapshots = Snapshot.objects.filter(id__in=snapshot_ids)
+    count = snapshots.count()
+
+    if count == 0:
+        rprint("[yellow]No matching snapshots found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} snapshots (dry run)[/yellow]", file=sys.stderr)
+        for snapshot in snapshots:
+            rprint(f"  [dim]{snapshot.id}[/dim] {snapshot.url[:60]}", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = snapshots.delete()
+    rprint(f"[green]Deleted {deleted_count} snapshots[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+
+@click.group()
+def main():
+    """Manage Snapshot records."""
+    pass
+
+
+@main.command("create")
+@click.argument("urls", nargs=-1)
+@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
+@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
+@click.option("--depth", "-d", type=int, default=0, help="Crawl depth (default: 0)")
+def create_cmd(urls: tuple, tag: str, status: str, depth: int):
+    """Create Snapshots from URLs or stdin JSONL."""
+    sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth))
+
+
+@main.command("list")
+@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
+@click.option("--url__icontains", help="Filter by URL contains")
+@click.option("--url__istartswith", help="Filter by URL starts with")
+@click.option("--tag", "-t", help="Filter by tag name")
+@click.option("--crawl-id", help="Filter by crawl ID")
+@click.option("--limit", "-n", type=int, help="Limit number of results")
+@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
+@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
+@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
+@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
+@click.argument("query", nargs=-1)
+def list_cmd(
+    status: str | None,
+    url__icontains: str | None,
+    url__istartswith: str | None,
+    tag: str | None,
+    crawl_id: str | None,
+    limit: int | None,
+    sort: str | None,
+    csv: str | None,
+    with_headers: bool,
+    search: str | None,
+    query: tuple[str, ...],
+):
+    """List Snapshots as JSONL."""
+    sys.exit(
+        list_snapshots(
+            status=status,
+            url__icontains=url__icontains,
+            url__istartswith=url__istartswith,
+            tag=tag,
+            crawl_id=crawl_id,
+            limit=limit,
+            sort=sort,
+            csv=csv,
+            with_headers=with_headers,
+            search=search,
+            query=" ".join(query),
+        ),
+    )
+
+
+@main.command("update")
+@click.option("--status", "-s", help="Set status")
+@click.option("--tag", "-t", help="Add tag")
+def update_cmd(status: str | None, tag: str | None):
+    """Update Snapshots from stdin JSONL."""
+    sys.exit(update_snapshots(status=status, tag=tag))
+
+
+@main.command("delete")
+@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
+@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
+def delete_cmd(yes: bool, dry_run: bool):
+    """Delete Snapshots from stdin JSONL."""
+    sys.exit(delete_snapshots(yes=yes, dry_run=dry_run))
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox snapshot"
+
+import sys
+
+import rich_click as click
+
+from archivebox.cli.archivebox_snapshot import create_snapshots
+
+
+@click.command(context_settings={"ignore_unknown_options": True})
+@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
+@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
+@click.option("--depth", "-d", type=int, default=0, help="Crawl depth (default: 0)")
+@click.argument("urls", nargs=-1)
+def main(tag: str, status: str, depth: int, urls: tuple[str, ...]):
+    """Backwards-compatible `archivebox snapshot URL...` entrypoint."""
+    sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth))
+
+
+if __name__ == "__main__":
+    main()
@@ -1,32 +1,139 @@
 #!/usr/bin/env python3

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox status'
+__package__ = "archivebox.cli"

-import sys
-import argparse
 from pathlib import Path
-from typing import Optional, List, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
-from ..main import status
+import rich_click as click
+from rich import print
+
+from archivebox.misc.util import enforce_types, docstring
+from archivebox.config import DATA_DIR, CONSTANTS, ARCHIVE_DIR
+from archivebox.config.common import SHELL_CONFIG
+from archivebox.misc.legacy import parse_json_links_details
+from archivebox.misc.system import get_dir_size
+from archivebox.misc.logging_util import printable_filesize


+@enforce_types
+def status(out_dir: Path = DATA_DIR) -> None:
+    """Print out some info and statistics about the archive collection"""
+
+    from django.contrib.auth import get_user_model
+    from django.db.models import Sum
+    from django.db.models.functions import Coalesce
+    from archivebox.core.models import Snapshot
+
+    User = get_user_model()
+
+    print("[green]\\[*] Scanning archive main index...[/green]")
+    print(f"[yellow]   {out_dir}/*[/yellow]")
+    num_bytes, num_dirs, num_files = get_dir_size(out_dir, recursive=False, pattern="index.")
+    size = printable_filesize(num_bytes)
+    print(f"    Index size: {size} across {num_files} files")
+    print()
+
+    links = list(Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)))
+    num_sql_links = len(links)
+    num_link_details = sum(1 for link in parse_json_links_details(out_dir=out_dir))
+    print(f"    > SQL Main Index: {num_sql_links} links".ljust(36), f"(found in {CONSTANTS.SQL_INDEX_FILENAME})")
+    print(f"    > JSON Link Details: {num_link_details} links".ljust(36), f"(found in {ARCHIVE_DIR.name}/*/index.json)")
+    print()
+    print("[green]\\[*] Scanning archive data directories...[/green]")
+    users_dir = out_dir / "users"
+    scan_roots = [root for root in (ARCHIVE_DIR, users_dir) if root.exists()]
+    scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(ARCHIVE_DIR)
+    print(f"[yellow]   {scan_roots_display}[/yellow]")
+    num_bytes = num_dirs = num_files = 0
+    for root in scan_roots:
+        root_bytes, root_dirs, root_files = get_dir_size(root)
+        num_bytes += root_bytes
+        num_dirs += root_dirs
+        num_files += root_files
+    size = printable_filesize(num_bytes)
+    print(f"    Size: {size} across {num_files} files in {num_dirs} directories")
+
+    # Use DB as source of truth for snapshot status
+    num_indexed = len(links)
+    num_archived = sum(1 for snapshot in links if snapshot.is_archived)
+    num_unarchived = max(num_indexed - num_archived, 0)
+    print(f"    > indexed: {num_indexed}".ljust(36), "(total snapshots in DB)")
+    print(f"      > archived: {num_archived}".ljust(36), "(snapshots with archived content)")
+    print(f"      > unarchived: {num_unarchived}".ljust(36), "(snapshots pending archiving)")
+
+    # Count snapshot directories on filesystem across both legacy and current layouts.
+    expected_snapshot_dirs = {str(Path(snapshot.output_dir).resolve()) for snapshot in links if Path(snapshot.output_dir).exists()}
+    discovered_snapshot_dirs = set()
+
+    if ARCHIVE_DIR.exists():
+        discovered_snapshot_dirs.update(str(entry.resolve()) for entry in ARCHIVE_DIR.iterdir() if entry.is_dir())
+
+    if users_dir.exists():
+        discovered_snapshot_dirs.update(str(entry.resolve()) for entry in users_dir.glob("*/snapshots/*/*/*") if entry.is_dir())
+
+    orphaned_dirs = sorted(discovered_snapshot_dirs - expected_snapshot_dirs)
+    num_present = len(discovered_snapshot_dirs)
+    num_valid = len(discovered_snapshot_dirs & expected_snapshot_dirs)
+    print()
+    print(f"    > present: {num_present}".ljust(36), "(snapshot directories on disk)")
+    print(f"      > [green]valid:[/green] {num_valid}".ljust(36), "               (directories with matching DB entry)")
+
+    num_orphaned = len(orphaned_dirs)
+    print(f"      > [red]orphaned:[/red] {num_orphaned}".ljust(36), "         (directories without matching DB entry)")
+
+    if num_indexed:
+        print("    [violet]Hint:[/violet] You can list snapshots by status like so:")
+        print("        [green]archivebox list --status=<status>  (e.g. archived, queued, etc.)[/green]")
+
+    if orphaned_dirs:
+        print("    [violet]Hint:[/violet] To automatically import orphaned data directories into the main index, run:")
+        print("        [green]archivebox init[/green]")
+
+    print()
+    print("[green]\\[*] Scanning recent archive changes and user logins:[/green]")
+    print(f"[yellow]   {CONSTANTS.LOGS_DIR}/*[/yellow]")
+    admin_users = User.objects.filter(is_superuser=True).exclude(username="system")
+    users = [user.get_username() for user in admin_users]
+    print(f"    UI users {len(users)}: {', '.join(users)}")
+    last_login = admin_users.order_by("last_login").last()
+    if last_login:
+        print(f"    Last UI login: {last_login.get_username()} @ {str(last_login.last_login)[:16]}")
+    last_downloaded = Snapshot.objects.order_by("downloaded_at").last()
+    if last_downloaded:
+        print(f"    Last changes: {str(last_downloaded.downloaded_at)[:16]}")
+
+    if not users:
+        print()
+        print("    [violet]Hint:[/violet] You can create an admin user by running:")
+        print("        [green]archivebox manage createsuperuser[/green]")
+
+    print()
+    recent_snapshots = sorted(
+        links,
+        key=lambda snapshot: snapshot.downloaded_at or snapshot.modified_at or snapshot.created_at,
+        reverse=True,
+    )[:10]
+    for snapshot in recent_snapshots:
+        if not snapshot.downloaded_at:
+            continue
+        print(
+            (
+                "[grey53] "
+                f"   > {str(snapshot.downloaded_at)[:16]} "
+                f"[{snapshot.num_outputs} {('X', '√')[snapshot.is_archived]} {printable_filesize(snapshot.archive_size)}] "
+                f'"{snapshot.title}": {snapshot.url}'
+                "[/grey53]"
+            )[: SHELL_CONFIG.TERM_WIDTH],
+        )
+    print("[grey53]   ...")
+
+
+@click.command()
@docstring(status.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=status.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.parse_args(args or ())
-    reject_stdin(__command__, stdin)
-
-    status(out_dir=Path(pwd) if pwd else DATA_DIR)
+def main(**kwargs):
+    """Print out some info and statistics about the archive collection"""
+    status(**kwargs)


-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,299 @@
+#!/usr/bin/env python3
+
+"""
+archivebox tag <action> [args...] [--filters]
+
+Manage Tag records.
+
+Actions:
+    create  - Create Tags
+    list    - List Tags as JSONL (with optional filters)
+    update  - Update Tags from stdin JSONL
+    delete  - Delete Tags from stdin JSONL
+
+Examples:
+    # Create
+    archivebox tag create news tech science
+    archivebox tag create "important stuff"
+
+    # List
+    archivebox tag list
+    archivebox tag list --name__icontains=news
+
+    # Update (rename tags)
+    archivebox tag list --name=oldname | archivebox tag update --name=newname
+
+    # Delete
+    archivebox tag list --name=unused | archivebox tag delete --yes
+"""
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox tag"
+
+import sys
+from collections.abc import Iterable
+
+import rich_click as click
+from rich import print as rprint
+
+from archivebox.cli.cli_utils import apply_filters
+
+
+# =============================================================================
+# CREATE
+# =============================================================================
+
+
+def create_tags(names: Iterable[str]) -> int:
+    """
+    Create Tags from names.
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.core.models import Tag
+
+    is_tty = sys.stdout.isatty()
+
+    # Convert to list if needed
+    name_list = list(names) if names else []
+
+    if not name_list:
+        rprint("[yellow]No tag names provided. Pass names as arguments.[/yellow]", file=sys.stderr)
+        return 1
+
+    created_count = 0
+    for name in name_list:
+        name = name.strip()
+        if not name:
+            continue
+
+        tag, created = Tag.objects.get_or_create(name=name)
+
+        if not is_tty:
+            write_record(tag.to_json())
+
+        if created:
+            created_count += 1
+            rprint(f"[green]Created tag: {name}[/green]", file=sys.stderr)
+        else:
+            rprint(f"[dim]Tag already exists: {name}[/dim]", file=sys.stderr)
+
+    rprint(f"[green]Created {created_count} new tags[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+
+def list_tags(
+    name: str | None = None,
+    name__icontains: str | None = None,
+    limit: int | None = None,
+) -> int:
+    """
+    List Tags as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.core.models import Tag
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Tag.objects.all().order_by("name")
+
+    # Apply filters
+    filter_kwargs = {
+        "name": name,
+        "name__icontains": name__icontains,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for tag in queryset:
+        snapshot_count = tag.snapshot_set.count()
+        if is_tty:
+            rprint(f"[cyan]{tag.name:30}[/cyan] [dim]({snapshot_count} snapshots)[/dim]")
+        else:
+            write_record(tag.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} tags[/dim]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# UPDATE
+# =============================================================================
+
+
+def update_tags(name: str | None = None) -> int:
+    """
+    Update Tags from stdin JSONL.
+
+    Reads Tag records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from archivebox.misc.jsonl import read_stdin, write_record
+    from archivebox.core.models import Tag
+
+    is_tty = sys.stdout.isatty()
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        tag_id = record.get("id")
+        old_name = record.get("name")
+
+        if not tag_id and not old_name:
+            continue
+
+        try:
+            if tag_id:
+                tag = Tag.objects.get(id=tag_id)
+            else:
+                tag = Tag.objects.get(name=old_name)
+
+            # Apply updates from CLI flags
+            if name:
+                tag.name = name
+                tag.save()
+
+            updated_count += 1
+
+            if not is_tty:
+                write_record(tag.to_json())
+
+        except Tag.DoesNotExist:
+            rprint(f"[yellow]Tag not found: {tag_id or old_name}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} tags[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# DELETE
+# =============================================================================
+
+
+def delete_tags(yes: bool = False, dry_run: bool = False) -> int:
+    """
+    Delete Tags from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
+    from archivebox.core.models import Tag
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    # Collect tag IDs or names
+    tag_ids = []
+    tag_names = []
+    for r in records:
+        if r.get("id"):
+            tag_ids.append(r["id"])
+        elif r.get("name"):
+            tag_names.append(r["name"])
+
+    if not tag_ids and not tag_names:
+        rprint("[yellow]No valid tag IDs or names in input[/yellow]", file=sys.stderr)
+        return 1
+
+    from django.db.models import Q
+
+    query = Q()
+    if tag_ids:
+        query |= Q(id__in=tag_ids)
+    if tag_names:
+        query |= Q(name__in=tag_names)
+
+    tags = Tag.objects.filter(query)
+    count = tags.count()
+
+    if count == 0:
+        rprint("[yellow]No matching tags found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} tags (dry run)[/yellow]", file=sys.stderr)
+        for tag in tags:
+            rprint(f"  {tag.name}", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = tags.delete()
+    rprint(f"[green]Deleted {deleted_count} tags[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+
+@click.group()
+def main():
+    """Manage Tag records."""
+    pass
+
+
+@main.command("create")
+@click.argument("names", nargs=-1)
+def create_cmd(names: tuple):
+    """Create Tags from names."""
+    sys.exit(create_tags(names))
+
+
+@main.command("list")
+@click.option("--name", help="Filter by exact name")
+@click.option("--name__icontains", help="Filter by name contains")
+@click.option("--limit", "-n", type=int, help="Limit number of results")
+def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
+    """List Tags as JSONL."""
+    sys.exit(list_tags(name=name, name__icontains=name__icontains, limit=limit))
+
+
+@main.command("update")
+@click.option("--name", "-n", help="Set new name")
+def update_cmd(name: str | None):
+    """Update Tags from stdin JSONL."""
+    sys.exit(update_tags(name=name))
+
+
+@main.command("delete")
+@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
+@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
+def delete_cmd(yes: bool, dry_run: bool):
+    """Delete Tags from stdin JSONL."""
+    sys.exit(delete_tags(yes=yes, dry_run=dry_run))
+
+
+if __name__ == "__main__":
+    main()
@@ -1,136 +1,600 @@
 #!/usr/bin/env python3

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox update'
+__package__ = "archivebox.cli"

-import sys
-import argparse
+import os
+import time
+
+from typing import TYPE_CHECKING, Any
+from collections.abc import Callable, Iterable
 from pathlib import Path
-from typing import List, Optional, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..index import (
-    LINK_FILTERS,
-    get_indexed_folders,
-    get_archived_folders,
-    get_unarchived_folders,
-    get_present_folders,
-    get_valid_folders,
-    get_invalid_folders,
-    get_duplicate_folders,
-    get_orphaned_folders,
-    get_corrupted_folders,
-    get_unrecognized_folders,
-)
-from ..logging_util import SmartFormatter, accept_stdin
-from ..main import update
+import rich_click as click
+from django.core.exceptions import ObjectDoesNotExist
+from django.db.models import Q, QuerySet
+
+from archivebox.misc.util import enforce_types, docstring
+
+if TYPE_CHECKING:
+    from archivebox.core.models import Snapshot
+    from archivebox.crawls.models import Crawl


-@docstring(update.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=update.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
+LINK_FILTERS: dict[str, Callable[[str], Q]] = {
+    "exact": lambda pattern: Q(url=pattern),
+    "substring": lambda pattern: Q(url__icontains=pattern),
+    "regex": lambda pattern: Q(url__iregex=pattern),
+    "domain": lambda pattern: (
+        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
+    ),
+    "tag": lambda pattern: Q(tags__name=pattern),
+    "timestamp": lambda pattern: Q(timestamp=pattern),
+}
+
+
+def _apply_pattern_filters(
+    snapshots: QuerySet["Snapshot", "Snapshot"],
+    filter_patterns: list[str],
+    filter_type: str,
+) -> QuerySet["Snapshot", "Snapshot"]:
+    filter_builder = LINK_FILTERS.get(filter_type)
+    if filter_builder is None:
+        raise SystemExit(2)
+
+    query = Q()
+    for pattern in filter_patterns:
+        query |= filter_builder(pattern)
+    return snapshots.filter(query)
+
+
+def _get_snapshot_crawl(snapshot: "Snapshot") -> "Crawl | None":
+    try:
+        return snapshot.crawl
+    except ObjectDoesNotExist:
+        return None
+
+
+def _get_search_indexing_plugins() -> list[str]:
+    from abx_dl.models import discover_plugins
+    from archivebox.hooks import get_search_backends
+
+    available_backends = set(get_search_backends())
+    plugins = discover_plugins()
+    return sorted(
+        plugin_name
+        for plugin_name, plugin in plugins.items()
+        if plugin_name.startswith("search_backend_")
+        and plugin_name.removeprefix("search_backend_") in available_backends
+        and any("Snapshot" in hook.name and "index" in hook.name.lower() for hook in plugin.hooks)
    )
-    parser.add_argument(
-        '--only-new', #'-n',
-        action='store_true',
-        help="Don't attempt to retry previously skipped/failed links when updating",
+
+
+def _build_filtered_snapshots_queryset(
+    *,
+    filter_patterns: Iterable[str],
+    filter_type: str,
+    before: float | None,
+    after: float | None,
+    resume: str | None = None,
+):
+    from archivebox.core.models import Snapshot
+    from datetime import datetime
+
+    snapshots = Snapshot.objects.all()
+
+    if filter_patterns:
+        snapshots = _apply_pattern_filters(snapshots, list(filter_patterns), filter_type)
+
+    if before:
+        snapshots = snapshots.filter(bookmarked_at__lt=datetime.fromtimestamp(before))
+    if after:
+        snapshots = snapshots.filter(bookmarked_at__gt=datetime.fromtimestamp(after))
+    if resume:
+        snapshots = snapshots.filter(timestamp__lte=resume)
+
+    return snapshots.select_related("crawl").order_by("-bookmarked_at")
+
+
+def reindex_snapshots(
+    snapshots: QuerySet["Snapshot", "Snapshot"],
+    *,
+    search_plugins: list[str],
+    batch_size: int,
+) -> dict[str, int]:
+    from archivebox.cli.archivebox_extract import run_plugins
+
+    stats = {"processed": 0, "reconciled": 0, "queued": 0, "reindexed": 0}
+    records: list[dict[str, str]] = []
+
+    total = snapshots.count()
+    print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}")
+
+    for snapshot in snapshots.iterator(chunk_size=batch_size):
+        stats["processed"] += 1
+
+        if _get_snapshot_crawl(snapshot) is None:
+            continue
+
+        output_dir = Path(snapshot.output_dir)
+        has_directory = output_dir.exists() and output_dir.is_dir()
+        if has_directory:
+            snapshot.reconcile_with_index_json()
+            stats["reconciled"] += 1
+
+        for plugin_name in search_plugins:
+            existing_result = snapshot.archiveresult_set.filter(plugin=plugin_name).order_by("-created_at").first()
+            if existing_result:
+                existing_result.reset_for_retry()
+            records.append(
+                {
+                    "type": "ArchiveResult",
+                    "snapshot_id": str(snapshot.id),
+                    "plugin": plugin_name,
+                },
+            )
+            stats["queued"] += 1
+
+    if not records:
+        return stats
+
+    exit_code = run_plugins(
+        args=(),
+        records=records,
+        wait=True,
+        emit_results=False,
    )
-    parser.add_argument(
-        '--index-only', #'-o',
-        action='store_true',
-        help="Update the main index without archiving any content",
-    )
-    parser.add_argument(
-        '--resume', #'-r',
-        type=float,
-        help='Resume the update process from a given timestamp',
-        default=None,
-    )
-    parser.add_argument(
-        '--overwrite', #'-x',
-        action='store_true',
-        help='Ignore existing archived content and overwrite with new versions (DANGEROUS)',
-    )
-    parser.add_argument(
-        '--before', #'-b',
-        type=float,
-        help="Update only links bookmarked before the given timestamp.",
-        default=None,
-    )
-    parser.add_argument(
-        '--after', #'-a',
-        type=float,
-        help="Update only links bookmarked after the given timestamp.",
-        default=None,
-    )
-    parser.add_argument(
-        '--status',
-        type=str,
-        choices=('indexed', 'archived', 'unarchived', 'present', 'valid', 'invalid', 'duplicate', 'orphaned', 'corrupted', 'unrecognized'),
-        default='indexed',
-        help=(
-            'Update only links or data directories that have the given status\n'
-            f'    indexed       {get_indexed_folders.__doc__} (the default)\n'
-            f'    archived      {get_archived_folders.__doc__}\n'
-            f'    unarchived    {get_unarchived_folders.__doc__}\n'
-            '\n'
-            f'    present       {get_present_folders.__doc__}\n'
-            f'    valid         {get_valid_folders.__doc__}\n'
-            f'    invalid       {get_invalid_folders.__doc__}\n'
-            '\n'
-            f'    duplicate     {get_duplicate_folders.__doc__}\n'
-            f'    orphaned      {get_orphaned_folders.__doc__}\n'
-            f'    corrupted     {get_corrupted_folders.__doc__}\n'
-            f'    unrecognized  {get_unrecognized_folders.__doc__}\n'
+    if exit_code != 0:
+        raise SystemExit(exit_code)
+
+    stats["reindexed"] = len(records)
+    return stats
+
+
+@enforce_types
+def update(
+    filter_patterns: Iterable[str] = (),
+    filter_type: str = "exact",
+    before: float | None = None,
+    after: float | None = None,
+    resume: str | None = None,
+    batch_size: int = 100,
+    continuous: bool = False,
+    index_only: bool = False,
+) -> None:
+    """
+    Update snapshots: migrate old dirs, reconcile DB, and re-queue for archiving.
+
+    Three-phase operation (without filters):
+    - Phase 1: Drain old archive/ dirs by moving to new fs location (0.8.x → 0.9.x)
+    - Phase 2: O(n) scan over entire DB from most recent to least recent
+    - No orphan scans needed (trust 1:1 mapping between DB and filesystem after phase 1)
+
+    With filters: Only phase 2 (DB query), no filesystem operations.
+    Without filters: All phases (full update).
+    """
+
+    from rich import print
+    from archivebox.config.django import setup_django
+
+    setup_django()
+
+    from django.core.management import call_command
+
+    # Run migrations first to ensure DB schema is up-to-date
+    print("[*] Checking for pending migrations...")
+    try:
+        call_command("migrate", "--no-input", verbosity=0)
+    except Exception as e:
+        print(f"[!] Warning: Migration check failed: {e}")
+
+    while True:
+        if index_only:
+            search_plugins = _get_search_indexing_plugins()
+            if not search_plugins:
+                print("[*] No search indexing plugins are available, nothing to backfill.")
+                break
+
+            if not (filter_patterns or before or after):
+                print("[*] Phase 1: Draining old archive/ directories (0.8.x → 0.9.x migration)...")
+                drain_old_archive_dirs(
+                    resume_from=resume,
+                    batch_size=batch_size,
+                )
+
+            snapshots = _build_filtered_snapshots_queryset(
+                filter_patterns=filter_patterns,
+                filter_type=filter_type,
+                before=before,
+                after=after,
+                resume=resume,
+            )
+            stats = reindex_snapshots(
+                snapshots,
+                search_plugins=search_plugins,
+                batch_size=batch_size,
+            )
+            print_index_stats(stats)
+        elif filter_patterns or before or after:
+            # Filtered mode: query DB only
+            print("[*] Processing filtered snapshots from database...")
+            stats = process_filtered_snapshots(
+                filter_patterns=filter_patterns,
+                filter_type=filter_type,
+                before=before,
+                after=after,
+                resume=resume,
+                batch_size=batch_size,
+            )
+            print_stats(stats)
+        else:
+            # Full mode: drain old dirs + process DB
+            stats_combined = {"phase1": {}, "phase2": {}}
+
+            print("[*] Phase 1: Draining old archive/ directories (0.8.x → 0.9.x migration)...")
+            stats_combined["phase1"] = drain_old_archive_dirs(
+                resume_from=resume,
+                batch_size=batch_size,
+            )
+
+            print("[*] Phase 2: Processing all database snapshots (most recent first)...")
+            stats_combined["phase2"] = process_all_db_snapshots(batch_size=batch_size, resume=resume)
+
+            # Phase 3: Deduplication (disabled for now)
+            # print('[*] Phase 3: Deduplicating...')
+            # stats_combined['deduplicated'] = Snapshot.find_and_merge_duplicates()
+
+            print_combined_stats(stats_combined)
+
+        if not continuous:
+            break
+
+        print("[yellow]Sleeping 60s before next pass...[/yellow]")
+        time.sleep(60)
+        resume = None
+
+
+def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100) -> dict[str, int]:
+    """
+    Drain old archive/ directories (0.8.x → 0.9.x migration).
+
+    Only processes real directories (skips symlinks - those are already migrated).
+    For each old dir found in archive/:
+      1. Load or create DB snapshot
+      2. Trigger fs migration on save() to move to data/users/{user}/...
+      3. Leave symlink in archive/ pointing to new location
+
+    After this drains, archive/ should only contain symlinks and we can trust
+    1:1 mapping between DB and filesystem.
+    """
+    from archivebox.core.models import Snapshot
+    from archivebox.config import CONSTANTS
+    from django.db import transaction
+
+    stats = {"processed": 0, "migrated": 0, "skipped": 0, "invalid": 0}
+
+    archive_dir = CONSTANTS.ARCHIVE_DIR
+    if not archive_dir.exists():
+        return stats
+
+    print("[DEBUG Phase1] Scanning for old directories in archive/...")
+
+    # Scan for real directories only (skip symlinks - they're already migrated)
+    all_entries = list(os.scandir(archive_dir))
+    print(f"[DEBUG Phase1] Total entries in archive/: {len(all_entries)}")
+    entries = [
+        (e.stat().st_mtime, e.path)
+        for e in all_entries
+        if e.is_dir(follow_symlinks=False)  # Skip symlinks
+    ]
+    entries.sort(reverse=True)  # Newest first
+    print(f"[DEBUG Phase1] Real directories (not symlinks): {len(entries)}")
+    print(f"[*] Found {len(entries)} old directories to drain")
+
+    for mtime, entry_path in entries:
+        entry_path = Path(entry_path)
+
+        # Resume from timestamp if specified
+        if resume_from and entry_path.name > resume_from:
+            continue
+
+        stats["processed"] += 1
+
+        # Try to load existing snapshot from DB
+        snapshot = Snapshot.load_from_directory(entry_path)
+
+        if not snapshot:
+            # Not in DB - create new snapshot record
+            snapshot = Snapshot.create_from_directory(entry_path)
+            if not snapshot:
+                # Invalid directory - move to invalid/
+                Snapshot.move_directory_to_invalid(entry_path)
+                stats["invalid"] += 1
+                print(f"    [{stats['processed']}] Invalid: {entry_path.name}")
+                continue
+
+            try:
+                snapshot.save()
+                stats["migrated"] += 1
+                print(f"    [{stats['processed']}] Imported orphaned snapshot: {entry_path.name}")
+            except Exception as e:
+                stats["skipped"] += 1
+                print(f"    [{stats['processed']}] Skipped (error: {e}): {entry_path.name}")
+            continue
+
+        # Ensure snapshot has a valid crawl (migration 0024 may have failed)
+        has_valid_crawl = _get_snapshot_crawl(snapshot) is not None
+
+        if not has_valid_crawl:
+            # Create a new crawl (created_by will default to system user)
+            from archivebox.crawls.models import Crawl
+
+            crawl = Crawl.objects.create(urls=snapshot.url)
+            # Use queryset update to avoid triggering save() hooks
+            from archivebox.core.models import Snapshot as SnapshotModel
+
+            SnapshotModel.objects.filter(pk=snapshot.pk).update(crawl=crawl)
+            # Refresh the instance
+            snapshot.crawl = crawl
+            print(f"[DEBUG Phase1] Created missing crawl for snapshot {str(snapshot.id)[:8]}")
+
+        # Check if needs migration (0.8.x → 0.9.x)
+        print(
+            f"[DEBUG Phase1] Snapshot {str(snapshot.id)[:8]}: fs_version={snapshot.fs_version}, needs_migration={snapshot.fs_migration_needed}",
        )
-    )
-    parser.add_argument(
-        '--filter-type', '-t',
-        type=str,
-        choices=(*LINK_FILTERS.keys(), 'search'),
-        default='exact',
-        help='Type of pattern matching to use when filtering URLs',
-    )
-    parser.add_argument(
-        'filter_patterns',
-        nargs='*',
-        type=str,
-        default=None,
-        help='Update only URLs matching these filter patterns.'
-    )
-    parser.add_argument(
-        "--extract",
-        type=str,
-        help="Pass a list of the extractors to be used. If the method name is not correct, it will be ignored. \
-              This does not take precedence over the configuration",
-        default=""
-    )
-    command = parser.parse_args(args or ())
+        if snapshot.fs_migration_needed:
+            try:
+                # Calculate paths using actual directory (entry_path), not snapshot.timestamp
+                # because snapshot.timestamp might be truncated
+                old_dir = entry_path
+                new_dir = snapshot.get_storage_path_for_version("0.9.0")
+                print(f"[DEBUG Phase1] Migrating {old_dir.name} → {new_dir}")

-    filter_patterns_str = None
-    if not command.filter_patterns:
-        filter_patterns_str = accept_stdin(stdin)
+                # Manually migrate files
+                if not new_dir.exists() and old_dir.exists():
+                    new_dir.mkdir(parents=True, exist_ok=True)
+                    import shutil

-    update(
-        resume=command.resume,
-        only_new=command.only_new,
-        index_only=command.index_only,
-        overwrite=command.overwrite,
-        filter_patterns_str=filter_patterns_str,
-        filter_patterns=command.filter_patterns,
-        filter_type=command.filter_type,
-        status=command.status,
-        after=command.after,
-        before=command.before,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
-        extractors=command.extract,
+                    file_count = 0
+                    for old_file in old_dir.rglob("*"):
+                        if old_file.is_file():
+                            rel_path = old_file.relative_to(old_dir)
+                            new_file = new_dir / rel_path
+                            if not new_file.exists():
+                                new_file.parent.mkdir(parents=True, exist_ok=True)
+                                shutil.copy2(old_file, new_file)
+                                file_count += 1
+                    print(f"[DEBUG Phase1] Copied {file_count} files")
+
+                # Update only fs_version field using queryset update (bypasses validation)
+                from archivebox.core.models import Snapshot as SnapshotModel
+
+                SnapshotModel.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0")
+
+                # Commit the transaction
+                transaction.commit()
+
+                # Cleanup: delete old dir and create symlink
+                if old_dir.exists() and old_dir != new_dir:
+                    snapshot._cleanup_old_migration_dir(old_dir, new_dir)
+
+                stats["migrated"] += 1
+                print(f"    [{stats['processed']}] Migrated: {entry_path.name}")
+            except Exception as e:
+                stats["skipped"] += 1
+                print(f"    [{stats['processed']}] Skipped (error: {e}): {entry_path.name}")
+        else:
+            stats["skipped"] += 1
+
+        if stats["processed"] % batch_size == 0:
+            transaction.commit()
+
+    transaction.commit()
+    return stats
+
+
+def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) -> dict[str, int]:
+    """
+    O(n) scan over entire DB from most recent to least recent.
+
+    For each snapshot:
+      1. Reconcile index.json with DB (merge titles, tags, archive results)
+      2. Queue for archiving (state machine will handle it)
+
+    No orphan detection needed - we trust 1:1 mapping between DB and filesystem
+    after Phase 1 has drained all old archive/ directories.
+    """
+    from archivebox.core.models import Snapshot
+    from django.db import transaction
+    from django.utils import timezone
+
+    stats = {"processed": 0, "reconciled": 0, "queued": 0}
+
+    queryset = Snapshot.objects.all()
+    if resume:
+        queryset = queryset.filter(timestamp__lte=resume)
+    total = queryset.count()
+    print(f"[*] Processing {total} snapshots from database (most recent first)...")
+
+    # Process from most recent to least recent
+    for snapshot in queryset.select_related("crawl").order_by("-bookmarked_at").iterator(chunk_size=batch_size):
+        stats["processed"] += 1
+
+        # Skip snapshots with missing crawl references (orphaned by migration errors)
+        if _get_snapshot_crawl(snapshot) is None:
+            continue
+
+        try:
+            print(
+                f"[DEBUG Phase2] Snapshot {str(snapshot.id)[:8]}: fs_version={snapshot.fs_version}, needs_migration={snapshot.fs_migration_needed}",
+            )
+
+            # Check if snapshot has a directory on disk
+            from pathlib import Path
+
+            output_dir = Path(snapshot.output_dir)
+            has_directory = output_dir.exists() and output_dir.is_dir()
+
+            # Only reconcile if directory exists (don't create empty directories for orphans)
+            if has_directory:
+                snapshot.reconcile_with_index_json()
+
+            # Clean up invalid field values from old migrations
+            if not isinstance(snapshot.current_step, int):
+                snapshot.current_step = 0
+
+            # If still needs migration, it's an orphan (no directory on disk)
+            # Mark it as migrated to prevent save() from triggering filesystem migration
+            if snapshot.fs_migration_needed:
+                if has_directory:
+                    print(f"[DEBUG Phase2] WARNING: Snapshot {str(snapshot.id)[:8]} has directory but still needs migration")
+                else:
+                    print(f"[DEBUG Phase2] Orphan snapshot {str(snapshot.id)[:8]} - marking as migrated without filesystem operation")
+                # Use queryset update to set fs_version without triggering save() hooks
+                from archivebox.core.models import Snapshot as SnapshotModel
+
+                SnapshotModel.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0")
+                snapshot.fs_version = "0.9.0"
+
+            # Queue for archiving (state machine will handle it)
+            snapshot.status = Snapshot.StatusChoices.QUEUED
+            snapshot.retry_at = timezone.now()
+            snapshot.save()
+
+            stats["reconciled"] += 1 if has_directory else 0
+            stats["queued"] += 1
+        except Exception as e:
+            # Skip snapshots that can't be processed (e.g., missing crawl)
+            print(f"    [!] Skipping snapshot {snapshot.id}: {e}")
+            continue
+
+        if stats["processed"] % batch_size == 0:
+            transaction.commit()
+            print(f"    [{stats['processed']}/{total}] Processed...")
+
+    transaction.commit()
+    return stats
+
+
+def process_filtered_snapshots(
+    filter_patterns: Iterable[str],
+    filter_type: str,
+    before: float | None,
+    after: float | None,
+    resume: str | None,
+    batch_size: int,
+) -> dict[str, int]:
+    """Process snapshots matching filters (DB query only)."""
+    from django.db import transaction
+    from django.utils import timezone
+
+    stats = {"processed": 0, "reconciled": 0, "queued": 0}
+
+    snapshots = _build_filtered_snapshots_queryset(
+        filter_patterns=filter_patterns,
+        filter_type=filter_type,
+        before=before,
+        after=after,
+        resume=resume,
    )
-    

-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    total = snapshots.count()
+    print(f"[*] Found {total} matching snapshots")
+
+    for snapshot in snapshots.select_related("crawl").iterator(chunk_size=batch_size):
+        stats["processed"] += 1
+
+        # Skip snapshots with missing crawl references
+        if _get_snapshot_crawl(snapshot) is None:
+            continue
+
+        try:
+            # Reconcile index.json with DB
+            snapshot.reconcile_with_index_json()
+
+            # Clean up invalid field values from old migrations
+            if not isinstance(snapshot.current_step, int):
+                snapshot.current_step = 0
+
+            # Queue for archiving
+            snapshot.status = Snapshot.StatusChoices.QUEUED
+            snapshot.retry_at = timezone.now()
+            snapshot.save()
+
+            stats["reconciled"] += 1
+            stats["queued"] += 1
+        except Exception as e:
+            # Skip snapshots that can't be processed
+            print(f"    [!] Skipping snapshot {snapshot.id}: {e}")
+            continue
+
+        if stats["processed"] % batch_size == 0:
+            transaction.commit()
+            print(f"    [{stats['processed']}/{total}] Processed...")
+
+    transaction.commit()
+    return stats
+
+
+def print_stats(stats: dict):
+    """Print statistics for filtered mode."""
+    from rich import print
+
+    print(f"""
+[green]Update Complete[/green]
+  Processed:   {stats["processed"]}
+  Reconciled:  {stats["reconciled"]}
+  Queued:      {stats["queued"]}
+""")
+
+
+def print_combined_stats(stats_combined: dict):
+    """Print statistics for full mode."""
+    from rich import print
+
+    s1 = stats_combined["phase1"]
+    s2 = stats_combined["phase2"]
+
+    print(f"""
+[green]Archive Update Complete[/green]
+
+Phase 1 (Drain Old Dirs):
+  Checked:     {s1.get("processed", 0)}
+  Migrated:    {s1.get("migrated", 0)}
+  Skipped:     {s1.get("skipped", 0)}
+  Invalid:     {s1.get("invalid", 0)}
+
+Phase 2 (Process DB):
+  Processed:   {s2.get("processed", 0)}
+  Reconciled:  {s2.get("reconciled", 0)}
+  Queued:      {s2.get("queued", 0)}
+""")
+
+
+def print_index_stats(stats: dict[str, Any]) -> None:
+    from rich import print
+
+    print(f"""
+[green]Search Reindex Complete[/green]
+  Processed:   {stats["processed"]}
+  Reconciled:  {stats["reconciled"]}
+  Queued:      {stats["queued"]}
+  Reindexed:   {stats["reindexed"]}
+""")
+
+
+@click.command()
+@click.option("--resume", type=str, help="Resume from timestamp")
+@click.option("--before", type=float, help="Only snapshots before timestamp")
+@click.option("--after", type=float, help="Only snapshots after timestamp")
+@click.option("--filter-type", "-t", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact")
+@click.option("--batch-size", type=int, default=100, help="Commit every N snapshots")
+@click.option("--continuous", is_flag=True, help="Run continuously as background worker")
+@click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content")
+@click.argument("filter_patterns", nargs=-1)
+@docstring(update.__doc__)
+def main(**kwargs):
+    update(**kwargs)
+
+
+if __name__ == "__main__":
+    main()
@@ -1,61 +1,241 @@
 #!/usr/bin/env python3

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox version'
+__package__ = "archivebox.cli"

 import sys
-import argparse
+import os
+import platform
 from pathlib import Path
-from typing import Optional, List, IO
+from collections.abc import Iterable

-# from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR, VERSION
-from ..logging_util import SmartFormatter, reject_stdin
+import rich_click as click
+
+from archivebox.misc.util import docstring, enforce_types


-# @docstring(version.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    """Print the ArchiveBox version and dependency information"""
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description="Print the ArchiveBox version and dependency information",   # version.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
+@enforce_types
+def version(
+    quiet: bool = False,
+    binaries: Iterable[str] = (),
+) -> list[str]:
+    """Print the ArchiveBox version, debug metadata, and installed dependency versions"""
+
+    # fast path for just getting the version and exiting, dont do any slower imports
+    from archivebox.config.version import VERSION
+
+    print(VERSION)
+    if quiet or "--version" in sys.argv:
+        return []
+
+    from rich.panel import Panel
+    from rich.console import Console
+
+    from archivebox.config import CONSTANTS, DATA_DIR
+    from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
+    from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID, IN_DOCKER
+    from archivebox.config.paths import get_data_locations, get_code_locations
+    from archivebox.config.common import SHELL_CONFIG, STORAGE_CONFIG, SEARCH_BACKEND_CONFIG
+    from archivebox.misc.logging_util import printable_folder_status
+    from archivebox.config.configset import get_config
+
+    console = Console()
+    prnt = console.print
+
+    # Check if LDAP is enabled (simple config lookup)
+    config = get_config()
+    LDAP_ENABLED = config.get("LDAP_ENABLED", False)
+
+    p = platform.uname()
+    COMMIT_HASH = get_COMMIT_HASH()
+    prnt(
+        f"[dark_green]ArchiveBox[/dark_green] [dark_goldenrod]v{CONSTANTS.VERSION}[/dark_goldenrod]",
+        f"COMMIT_HASH={COMMIT_HASH[:7] if COMMIT_HASH else 'unknown'}",
+        f"BUILD_TIME={get_BUILD_TIME()}",
    )
-    parser.add_argument(
-        '--quiet', '-q',
-        action='store_true',
-        help='Only print ArchiveBox version number and nothing else.',
-    )
-    parser.add_argument(
-        '--binproviders', '-p',
-        type=str,
-        help='Select binproviders to detect DEFAULT=env,apt,brew,sys_pip,venv_pip,lib_pip,pipx,sys_npm,lib_npm,puppeteer,playwright (all)',
-        default=None,
-    )
-    parser.add_argument(
-        '--binaries', '-b',
-        type=str,
-        help='Select binaries to detect DEFAULT=curl,wget,git,yt-dlp,chrome,single-file,readability-extractor,postlight-parser,... (all)',
-        default=None,
-    )
-    command = parser.parse_args(args or ())
-    reject_stdin(__command__, stdin)
-    
-    # for speed reasons, check if quiet flag was set and just return simple version immediately if so
-    if command.quiet:
-        print(VERSION)
-        return
-    
-    # otherwise do big expensive import to get the full version
-    from ..main import version
-    version(
-        quiet=command.quiet,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
-        binproviders=command.binproviders.split(',') if command.binproviders else None,
-        binaries=command.binaries.split(',') if command.binaries else None,
+    prnt(
+        f"IN_DOCKER={IN_DOCKER}",
+        f"IN_QEMU={SHELL_CONFIG.IN_QEMU}",
+        f"ARCH={p.machine}",
+        f"OS={p.system}",
+        f"PLATFORM={platform.platform()}",
+        f"PYTHON={sys.implementation.name.title()}" + (" (venv)" if CONSTANTS.IS_INSIDE_VENV else ""),
    )

+    try:
+        OUTPUT_IS_REMOTE_FS = get_data_locations().DATA_DIR.is_mount or get_data_locations().ARCHIVE_DIR.is_mount
+    except Exception:
+        OUTPUT_IS_REMOTE_FS = False

-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    try:
+        DATA_DIR_STAT = CONSTANTS.DATA_DIR.stat()
+        prnt(
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"FS_UID={DATA_DIR_STAT.st_uid}:{DATA_DIR_STAT.st_gid}",
+            f"FS_PERMS={STORAGE_CONFIG.OUTPUT_PERMISSIONS}",
+            f"FS_ATOMIC={STORAGE_CONFIG.ENFORCE_ATOMIC_WRITES}",
+            f"FS_REMOTE={OUTPUT_IS_REMOTE_FS}",
+        )
+    except Exception:
+        prnt(
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+        )
+
+    prnt(
+        f"DEBUG={SHELL_CONFIG.DEBUG}",
+        f"IS_TTY={SHELL_CONFIG.IS_TTY}",
+        f"SUDO={CONSTANTS.IS_ROOT}",
+        f"ID={CONSTANTS.MACHINE_ID}:{CONSTANTS.COLLECTION_ID}",
+        f"SEARCH_BACKEND={SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}",
+        f"LDAP={LDAP_ENABLED}",
+    )
+    prnt()
+
+    if not (os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
+        PANEL_TEXT = "\n".join(
+            (
+                "",
+                "[violet]Hint:[/violet] [green]cd[/green] into a collection [blue]DATA_DIR[/blue] and run [green]archivebox version[/green] again...",
+                "      [grey53]OR[/grey53] run [green]archivebox init[/green] to create a new collection in the current dir.",
+                "",
+                "      [i][grey53](this is [red]REQUIRED[/red] if you are opening a Github Issue to get help)[/grey53][/i]",
+                "",
+            ),
+        )
+        prnt(
+            Panel(
+                PANEL_TEXT,
+                expand=False,
+                border_style="grey53",
+                title="[red]:exclamation: No collection [blue]DATA_DIR[/blue] is currently active[/red]",
+                subtitle="Full version info is only available when inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
+            ),
+        )
+        prnt()
+        return []
+
+    prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
+    failures = []
+
+    # Setup Django before importing models
+    try:
+        from archivebox.config.django import setup_django
+
+        setup_django()
+
+        from archivebox.machine.models import Machine, Binary
+
+        machine = Machine.current()
+
+        # Get all binaries from the database with timeout protection
+        all_installed = (
+            Binary.objects.filter(
+                machine=machine,
+            )
+            .exclude(abspath="")
+            .exclude(abspath__isnull=True)
+            .order_by("name")
+        )
+
+        if not all_installed.exists():
+            prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
+        else:
+            for installed in all_installed:
+                # Skip if user specified specific binaries and this isn't one
+                if binaries and installed.name not in binaries:
+                    continue
+
+                if installed.is_valid:
+                    display_path = installed.abspath.replace(str(DATA_DIR), ".").replace(str(Path("~").expanduser()), "~")
+                    version_str = (installed.version or "unknown")[:15]
+                    provider = (installed.binprovider or "env")[:8]
+                    prnt(
+                        "",
+                        "[green]√[/green]",
+                        "",
+                        installed.name.ljust(18),
+                        version_str.ljust(16),
+                        provider.ljust(8),
+                        display_path,
+                        overflow="ignore",
+                        crop=False,
+                    )
+                else:
+                    prnt("", "[red]X[/red]", "", installed.name.ljust(18), "[grey53]not installed[/grey53]", overflow="ignore", crop=False)
+                    failures.append(installed.name)
+
+        # Show hint if no binaries are installed yet
+        has_any_installed = Binary.objects.filter(machine=machine).exclude(abspath="").exists()
+        if not has_any_installed:
+            prnt()
+            prnt("", "[grey53]Run [green]archivebox install[/green] to detect and install dependencies.[/grey53]")
+
+    except Exception as e:
+        # Handle database errors gracefully (locked, missing, etc.)
+        prnt()
+        prnt("", f"[yellow]Warning: Could not query binaries from database: {e}[/yellow]")
+        prnt("", "[grey53]Run [green]archivebox init[/green] and [green]archivebox install[/green] to set up dependencies.[/grey53]")
+
+    if not binaries:
+        # Show code and data locations
+        prnt()
+        prnt("[deep_sky_blue3][i] Code locations:[/deep_sky_blue3]")
+        try:
+            for name, path in get_code_locations().items():
+                if isinstance(name, str) and isinstance(path, dict):
+                    prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
+        except Exception as e:
+            prnt(f"  [red]Error getting code locations: {e}[/red]")
+
+        prnt()
+        if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
+            prnt("[bright_yellow][i] Data locations:[/bright_yellow]")
+            try:
+                for name, path in get_data_locations().items():
+                    if isinstance(name, str) and isinstance(path, dict):
+                        prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
+            except Exception as e:
+                prnt(f"  [red]Error getting data locations: {e}[/red]")
+
+            try:
+                from archivebox.misc.checks import check_data_dir_permissions
+
+                check_data_dir_permissions()
+            except Exception:
+                pass
+        else:
+            prnt()
+            prnt("[red][i] Data locations:[/red] (not in a data directory)")
+
+    prnt()
+
+    if failures:
+        prnt("[red]Error:[/red] [yellow]Failed to detect the following binaries:[/yellow]")
+        prnt(f"      [red]{', '.join(failures)}[/red]")
+        prnt()
+        prnt("[violet]Hint:[/violet] To install missing binaries automatically, run:")
+        prnt("      [green]archivebox install[/green]")
+        prnt()
+    return failures
+
+
+@click.command()
+@click.option(
+    "--quiet",
+    "-q",
+    is_flag=True,
+    help="Only print ArchiveBox version number and nothing else. (equivalent to archivebox --version)",
+)
+@click.option(
+    "--binaries",
+    "-b",
+    help="Select binaries to detect DEFAULT=curl,wget,git,yt-dlp,chrome,single-file,readability-extractor,postlight-parser,... (all)",
+)
+@docstring(version.__doc__)
+def main(**kwargs):
+    failures = version(**kwargs)
+    if failures:
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,44 @@
+"""
+Shared CLI utilities for ArchiveBox commands.
+
+This module contains common utilities used across multiple CLI commands,
+extracted to avoid code duplication.
+"""
+
+__package__ = "archivebox.cli"
+
+
+def apply_filters(queryset, filter_kwargs: dict, limit: int | None = None):
+    """
+    Apply Django-style filters from CLI kwargs to a QuerySet.
+
+    Supports: --status=queued, --url__icontains=example, --id__in=uuid1,uuid2
+
+    Args:
+        queryset: Django QuerySet to filter
+        filter_kwargs: Dict of filter key-value pairs from CLI
+        limit: Optional limit on results
+
+    Returns:
+        Filtered QuerySet
+
+    Example:
+        queryset = Snapshot.objects.all()
+        filter_kwargs = {'status': 'queued', 'url__icontains': 'example.com'}
+        filtered = apply_filters(queryset, filter_kwargs, limit=10)
+    """
+    filters = {}
+    for key, value in filter_kwargs.items():
+        if value is None or key in ("limit", "offset"):
+            continue
+        # Handle CSV lists for __in filters
+        if key.endswith("__in") and isinstance(value, str):
+            value = [v.strip() for v in value.split(",")]
+        filters[key] = value
+
+    if filters:
+        queryset = queryset.filter(**filters)
+    if limit:
+        queryset = queryset[:limit]
+
+    return queryset